From 39c13ccac6341d3fcd3a52e3a333afba14949f1a Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 30 Jan 2023 12:48:39 +0000 Subject: [PATCH 01/73] add basic form of numbers Signed-off-by: Jim O'Regan --- .../text_normalization/pl/data/numbers/digit.tsv | 9 +++++++++ .../text_normalization/pl/data/numbers/hundreds.tsv | 9 +++++++++ .../text_normalization/pl/data/numbers/teen.tsv | 10 ++++++++++ .../text_normalization/pl/data/numbers/tens.tsv | 8 ++++++++ 4 files changed, 36 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv new file mode 100644 index 000000000..7efd57899 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +jeden 1 +dwa 2 +trzy 3 +cztery 4 +pięć 5 +sześć 6 +siedem 7 +osiem 8 +dziewięć 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv new file mode 100644 index 000000000..306f322bb --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv @@ -0,0 +1,9 @@ +sto 100 +dwieście 200 +trzysta 300 +czterysta 400 +pięćset 500 +sześćset 600 +siedemset 700 +osiemset 800 +dziewięćset 900 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv new file mode 100644 index 000000000..1d812db0b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +dziesięć 10 +jedenaście 11 +dwanaście 12 +trzynaście 13 +czternaście 14 +piętnaście 15 +szesnaście 16 +siedemnaście 17 +osiemnaście 18 +dziewiętnaście 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv new file mode 100644 index 000000000..ff25c90e5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +dwadzieścia 20 +trzydzieści 30 +czterdzieści 40 +pięćdziesiąt 50 +sześćdziesiąt 60 +siedemdziesiąt 70 +osiemdziesiąt 80 +dziewięćdziesiąt 90 From b010af3a53369c444cf20caada72ecd3fc81153a Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 13 Feb 2023 14:41:25 +0000 Subject: [PATCH 02/73] remove trailing zeros Signed-off-by: Jim O'Regan --- .../pl/data/numbers/hundreds.tsv | 18 +++++++++--------- .../pl/data/numbers/tens.tsv | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv index 306f322bb..6c29e26a0 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds.tsv @@ -1,9 +1,9 @@ -sto 100 -dwieście 200 -trzysta 300 -czterysta 400 -pięćset 500 -sześćset 600 -siedemset 700 -osiemset 800 -dziewięćset 900 +sto 1 +dwieście 2 +trzysta 3 +czterysta 4 +pięćset 5 +sześćset 6 +siedemset 7 +osiemset 8 +dziewięćset 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv index ff25c90e5..2bb0a3a43 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens.tsv @@ -1,8 +1,8 @@ -dwadzieścia 20 -trzydzieści 30 -czterdzieści 40 -pięćdziesiąt 50 -sześćdziesiąt 60 -siedemdziesiąt 70 -osiemdziesiąt 80 -dziewięćdziesiąt 90 +dwadzieścia 2 +trzydzieści 3 +czterdzieści 4 +pięćdziesiąt 5 +sześćdziesiąt 6 +siedemdziesiąt 7 +osiemdziesiąt 8 +dziewięćdziesiąt 9 From eed5109a37d997c85db9ac400b4f0081a81e9bde Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 14 Feb 2023 10:01:54 +0000 Subject: [PATCH 03/73] start from german Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 199 ++++++++++++++++++ .../text_normalization/pl/taggers/ordinal.py | 49 +++++ 2 files changed, 248 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/ordinal.py diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py new file mode 100644 index 000000000..8f1a13b0c --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -0,0 +1,199 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict + +import pynini +from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from pynini.lib import pynutil + +AND = "und" + + +def get_ties_digit(digit_path: str, tie_path: str) -> 'pynini.FstLike': + """ + getting all inverse normalizations for numbers between 21 - 100 + + Args: + digit_path: file to digit tsv + tie_path: file to tie tsv, e.g. 20, 30, etc. + Returns: + res: fst that converts numbers to their verbalization + """ + + digits = defaultdict(list) + ties = defaultdict(list) + for k, v in load_labels(digit_path): + digits[v].append(k) + digits["1"] = ["ein"] + + for k, v in load_labels(tie_path): + ties[v].append(k) + + d = [] + for i in range(21, 100): + s = str(i) + if s[1] == "0": + continue + + for di in digits[s[1]]: + for ti in ties[s[0]]: + word = di + f" {AND} " + ti + d.append((word, s)) + + res = pynini.string_map(d) + return res + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "101" -> cardinal { integer: "ein hundert und zehn" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digit_no_one = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + graph_one = pynini.string_file(get_abs_path("data/numbers/ones.tsv")).invert() + graph_digit = graph_digit_no_one | graph_one + self.digit = (graph_digit | graph_zero).optimize() + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).invert() + + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).invert() + # separator = "." + + def tens_no_zero(): + return ( + pynutil.delete("0") + graph_digit + | get_ties_digit( + get_abs_path("data/numbers/digit.tsv"), get_abs_path("data/numbers/ties.tsv") + ).invert() + | graph_teen + | (graph_ties + pynutil.delete("0")) + ) + + def hundred_non_zero(): + return (graph_digit_no_one + insert_space | pynini.cross("1", "ein ")) + pynutil.insert("hundert") + ( + pynini.closure(insert_space + pynutil.insert(AND, weight=0.0001), 0, 1) + insert_space + tens_no_zero() + | pynutil.delete("00") + ) | pynutil.delete("0") + tens_no_zero() + + def thousand(): + return (hundred_non_zero() + insert_space + pynutil.insert("tausend") | pynutil.delete("000")) + ( + insert_space + hundred_non_zero() | pynutil.delete("000") + ) + + optional_plural_quantity_en = pynini.closure(pynutil.insert("en", weight=-0.0001), 0, 1) + optional_plural_quantity_n = pynini.closure(pynutil.insert("n", weight=-0.0001), 0, 1) + graph_million = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("million") + optional_plural_quantity_en, + pynutil.delete("000"), + ) + + graph_billion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("milliarde") + optional_plural_quantity_n, + pynutil.delete("000"), + ) + + graph_trillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("billion") + optional_plural_quantity_en, + pynutil.delete("000"), + ) + + graph_quadrillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("billiarde") + optional_plural_quantity_n, + pynutil.delete("000"), + ) + + graph_quintillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("trillion") + optional_plural_quantity_en, + pynutil.delete("000"), + ) + + graph_sextillion = pynini.union( + hundred_non_zero() + insert_space + pynutil.insert("trilliarde") + optional_plural_quantity_n, + pynutil.delete("000"), + ) + graph = pynini.union( + graph_sextillion + + insert_space + + graph_quintillion + + insert_space + + graph_quadrillion + + insert_space + + graph_trillion + + insert_space + + graph_billion + + insert_space + + graph_million + + insert_space + + thousand() + ) + + fix_syntax = [ + ("eins tausend", "ein tausend"), + ("eins millionen", "eine million"), + ("eins milliarden", "eine milliarde"), + ("eins billionen", "eine billion"), + ("eins billiarden", "eine billiarde"), + ] + fix_syntax = pynini.union(*[pynini.cross(*x) for x in fix_syntax]) + self.graph = ( + ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT ** 24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) + @ pynini.cdrewrite(fix_syntax, "[BOS]", "", NEMO_SIGMA) + ) + self.graph |= graph_zero | pynini.cross("1", "eins") + + # self.graph = pynini.cdrewrite(pynutil.delete(separator), "", "", NEMO_SIGMA) @ self.graph + self.graph = self.graph.optimize() + + self.graph_hundred_component_at_least_one_none_zero_digit = ( + ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT ** 3 + @ hundred_non_zero() + ) | pynini.cross("1", "eins") + + self.graph_hundred_component_at_least_one_none_zero_digit = ( + self.graph_hundred_component_at_least_one_none_zero_digit.optimize() + ) + + self.two_digit_non_zero = ( + pynini.closure(NEMO_DIGIT, 1, 2) @ self.graph_hundred_component_at_least_one_none_zero_digit + ) + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py new file mode 100644 index 000000000..564466551 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Adapted from https://github.com/google/TextNormalizationCoveringGrammars +# Russian minimally supervised number grammar. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +from pynini.lib import pynutil + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "2." -> ordinal { integer: "zwei" } } + "2tes" -> ordinal { integer: "zwei" } } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic=False): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + endings = ["ter", "tes", "tem", "te", "ten"] + self.graph = ( + ( + pynini.closure(NEMO_DIGIT | pynini.accep(".")) + + pynutil.delete(pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")) + ) + @ cardinal_graph + ).optimize() + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() From 9e32ee8057316594898e005cd3474b54d0d3348a Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 14 Feb 2023 11:38:29 +0000 Subject: [PATCH 04/73] start of ordinal (adjective) inflection Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 564466551..17681a577 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -20,6 +20,49 @@ from pynini.lib import pynutil +def adjective_inflection(word: str): + def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): + if stem_b == "": + stem_b = stem + return { + "mi_sg_nom": mi_sg, + "mi_sg_gen": stem + "ego", + "mi_sg_dat": stem + "emu", + "mi_sg_ins": stem + vowel + "m", + "nt_sg_nom": stem + "e", + "f_sg_nom": stem_b + "a", + "f_sg_gen": stem + "ej", + "f_sg_ins": stem_b + "ą", + "mp_pl_nom": mp_pl, + "pl_ins": stem + vowel + "mi", + "pl_loc": stem + vowel + "ch", + + } + stem_b = "" + if word.endswith("en"): + stem = word[:-2] + "n" + mi_sg = word + mp_pl = stem + "i" + vowel = "y" + elif word.endswith("ni"): + stem = word + mi_sg = word + mp_pl = word + vowel = "" + elif word.endswith("szy"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "i" + vowel = "y" + elif word.endswith("gi"): + stem = word + stem_b = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "dzy" + vowel = "" + return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b) + + class OrdinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. From d3a9578996134868fbcc530dde2e8dd47431a95d Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 14 Feb 2023 14:32:41 +0000 Subject: [PATCH 05/73] remove some stuff Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 8f1a13b0c..102e230a6 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -25,43 +25,6 @@ ) from pynini.lib import pynutil -AND = "und" - - -def get_ties_digit(digit_path: str, tie_path: str) -> 'pynini.FstLike': - """ - getting all inverse normalizations for numbers between 21 - 100 - - Args: - digit_path: file to digit tsv - tie_path: file to tie tsv, e.g. 20, 30, etc. - Returns: - res: fst that converts numbers to their verbalization - """ - - digits = defaultdict(list) - ties = defaultdict(list) - for k, v in load_labels(digit_path): - digits[v].append(k) - digits["1"] = ["ein"] - - for k, v in load_labels(tie_path): - ties[v].append(k) - - d = [] - for i in range(21, 100): - s = str(i) - if s[1] == "0": - continue - - for di in digits[s[1]]: - for ti in ties[s[0]]: - word = di + f" {AND} " + ti - d.append((word, s)) - - res = pynini.string_map(d) - return res - class CardinalFst(GraphFst): """ From 336a9c37e303ea1908d275a7d9245998c7dc7d76 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 14 Feb 2023 14:32:51 +0000 Subject: [PATCH 06/73] some more on inflection Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 17681a577..b55fedc18 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -36,7 +36,7 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): "mp_pl_nom": mp_pl, "pl_ins": stem + vowel + "mi", "pl_loc": stem + vowel + "ch", - + "compound": stem_b + "o", } stem_b = "" if word.endswith("en"): @@ -44,7 +44,7 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): mi_sg = word mp_pl = stem + "i" vowel = "y" - elif word.endswith("ni"): + elif word[-2:] in ["ni", "ci"]: stem = word mi_sg = word mp_pl = word @@ -60,14 +60,24 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): mi_sg = word mp_pl = word[:-2] + "dzy" vowel = "" + elif word.endswith("sty"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-3] + "ści" + vowel = "y" + elif word.endswith("ty"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "ci" + vowel = "y" return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b) class OrdinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. - "2." -> ordinal { integer: "zwei" } } - "2tes" -> ordinal { integer: "zwei" } } + "2." -> ordinal { integer: "drugi" } } + "2-gi" -> ordinal { integer: "drugi" } } Args: cardinal: cardinal GraphFst From 51692bb6580c167d7987abb5efd3053bc0e8d2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20O=E2=80=99Regan?= Date: Tue, 4 Apr 2023 19:43:32 +0200 Subject: [PATCH 07/73] Update ordinal.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jim O’Regan --- .../text_normalization/pl/taggers/ordinal.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index b55fedc18..d827cd358 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Adapted from https://github.com/google/TextNormalizationCoveringGrammars -# Russian minimally supervised number grammar. - import pynini from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst from pynini.lib import pynutil @@ -80,16 +77,13 @@ class OrdinalFst(GraphFst): "2-gi" -> ordinal { integer: "drugi" } } Args: - cardinal: cardinal GraphFst deterministic: if True will provide a single transduction option, for False multiple transduction are generated (used for audio-based normalization) """ - def __init__(self, cardinal: GraphFst, deterministic=False): + def __init__(self, deterministic=False): super().__init__(name="ordinal", kind="classify", deterministic=deterministic) - cardinal_graph = cardinal.graph - endings = ["ter", "tes", "tem", "te", "ten"] self.graph = ( ( pynini.closure(NEMO_DIGIT | pynini.accep(".")) From 5ee82c2a9d249cae28acccfb4040a0506218a4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20O=E2=80=99Regan?= Date: Tue, 4 Apr 2023 19:46:31 +0200 Subject: [PATCH 08/73] Create digit.tsv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jim O’Regan --- .../text_normalization/pl/data/ordinal/digit.tsv | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv new file mode 100644 index 000000000..85a82bd4a --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/digit.tsv @@ -0,0 +1,9 @@ +pierwszy 1 +drugi 2 +trzeci 3 +czwarty 4 +piąty 5 +szósty 6 +siódmy 7 +ósmy 8 +dziewiąty 9 From cd4a8fedf6b8b24e368dd6aeb693371993870b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20O=E2=80=99Regan?= Date: Tue, 4 Apr 2023 19:48:07 +0200 Subject: [PATCH 09/73] Create teens.tsv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jim O’Regan --- .../text_normalization/pl/data/ordinal/teens.tsv | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv new file mode 100644 index 000000000..07b3478ec --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/teens.tsv @@ -0,0 +1,10 @@ +dziesiąty 10 +jedenasty 11 +dwunasty 12 +trzynasty 13 +czternasty 14 +piętnasty 15 +szesnasty 16 +siedemnasty 17 +osiemnasty 18 +dziewiętnasty 19 From d3037dd90f8d736a20e9ffcbb5e3437e5631f50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20O=E2=80=99Regan?= Date: Tue, 4 Apr 2023 19:49:05 +0200 Subject: [PATCH 10/73] Create tens.tsv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jim O’Regan --- .../text_normalization/pl/data/ordinal/tens.tsv | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv new file mode 100644 index 000000000..ad1795634 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/tens.tsv @@ -0,0 +1,8 @@ +dwudziesty 2 +trzydziesty 3 +czterdziesty 4 +pięćdziesiąty 5 +sześćdziesiąty 6 +siedemdziesiąty 7 +osiemdziesiąty 8 +dziewięćdziesiąty 9 From d36fe7392f3e57479f78feccfea34bbf1cb64daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20O=E2=80=99Regan?= Date: Tue, 4 Apr 2023 19:51:48 +0200 Subject: [PATCH 11/73] Create hundreds.tsv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jim O’Regan --- .../text_normalization/pl/data/ordinal/hundreds.tsv | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv new file mode 100644 index 000000000..6044ab8e4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/hundreds.tsv @@ -0,0 +1,9 @@ +setny 1 +dwusetny 2 +trzechsetny 3 +czterechsetny 4 +pięćsetny 5 +sześćsetny 6 +siedemsetny 7 +osiemsetny 8 +dziewięćsetny 9 From 95eded482d7e2d2bce1b71d0551bed8782240270 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 4 Apr 2023 18:22:50 +0000 Subject: [PATCH 12/73] add zero Signed-off-by: Jim O'Regan --- nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv | 1 + 1 file changed, 1 insertion(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv new file mode 100644 index 000000000..a1b116c10 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 From a0a4955b9aa27cbe3a2725960190805b0c2c8805 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 4 Apr 2023 18:23:06 +0000 Subject: [PATCH 13/73] replace with version from sv Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 381 +++++++++++++----- 1 file changed, 286 insertions(+), 95 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 102e230a6..45b826fab 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -1,4 +1,5 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022, 2023 Jim O'Regan for Språkbanken Tal # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,152 +12,342 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from collections import defaultdict - import pynini -from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, GraphFst, delete_space, insert_space, ) +from nemo_text_processing.text_normalization.sv.graph_utils import SV_ALPHA +from nemo_text_processing.text_normalization.sv.utils import get_abs_path from pynini.lib import pynutil +def make_million(number: str, non_zero_no_one: 'pynini.FstLike', deterministic: bool = True) -> 'pynini.FstLike': + """ + Helper function for millions/milliards and higher + Args: + number: the string of the number + non_zero_no_one: An fst of digits excluding 0 and 1, to prefix to the number + deterministic: if True, generate a deterministic fst + + Returns: + graph: A pynini.FstLike object + """ + old_orth = number.replace("lj", "lli") + graph = pynutil.add_weight(pynini.cross("001", number), -0.001) + if not deterministic: + graph |= pynutil.add_weight(pynini.cross("001", old_orth), -0.001) + # 'ett' is usually wrong for these numbers, but it occurs + for one in ["en", "ett"]: + graph |= pynutil.add_weight(pynini.cross("001", f"{one} {number}"), -0.001) + graph |= pynutil.add_weight(pynini.cross("001", f"{one} {old_orth}"), -0.001) + graph |= pynutil.add_weight(pynini.cross("001", f"{one}{number}"), -0.001) + graph |= pynutil.add_weight(pynini.cross("001", f"{one}{old_orth}"), -0.001) + graph |= non_zero_no_one + pynutil.insert(f" {number}er") + if not deterministic: + graph |= pynutil.add_weight(non_zero_no_one + pynutil.insert(f" {old_orth}er"), -0.001) + graph |= pynutil.add_weight(non_zero_no_one + pynutil.insert(f"{old_orth}er"), -0.001) + graph |= pynutil.delete("000") + graph += insert_space + return graph + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by space) + and converts to a string of digits: + "1 000" -> "1000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_separator = NEMO_SPACE + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. - "101" -> cardinal { integer: "ein hundert und zehn" } + Finite state transducer for classifying cardinals, e.g. + "1000" -> cardinal { integer: "tusen" } + "2 000 000" -> cardinal { integer: "två miljon" } Args: deterministic: if True will provide a single transduction option, for False multiple transduction are generated (used for audio-based normalization) """ - def __init__(self, deterministic: bool = False): + def __init__(self, deterministic: bool = True): super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) + ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) + hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + + # Any single digit + graph_digit = digit + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + self.digit = graph_digit + + single_digits_graph = graph_digit | zero + self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) + + # spoken this way, so useful for e2e ASR + alt_ties = ties @ pynini.cdrewrite(ties_alt_endings, "", "[EOS]", NEMO_SIGMA) + if not deterministic: + ties |= pynutil.add_weight(alt_ties, -0.001) + ties |= pynutil.add_weight(pynini.cross("4", "förtio"), -0.001) + ties |= pynutil.add_weight(pynini.cross("4", "förti"), -0.001) + + # Any double digit + graph_tens = teen + graph_ties = ties + if deterministic: + graph_tens |= graph_ties + (pynutil.delete('0') | graph_digit) + else: + graph_tens |= pynutil.add_weight(pynini.cross("18", "aderton"), -0.001) + graph_tens |= pynutil.add_weight( + graph_ties + (pynutil.delete('0') | (graph_digit | insert_space + graph_digit)), -0.001 + ) + + hundreds = digits_no_one + pynutil.insert("hundra") + hundreds |= pynini.cross("1", "hundra") + if not deterministic: + hundreds |= pynutil.add_weight(pynini.cross("1", "etthundra"), -0.001) + hundreds |= pynutil.add_weight(digit + pynutil.insert(NEMO_SPACE) + pynutil.insert("hundra"), -0.001) - graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() - graph_digit_no_one = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() - graph_one = pynini.string_file(get_abs_path("data/numbers/ones.tsv")).invert() - graph_digit = graph_digit_no_one | graph_one - self.digit = (graph_digit | graph_zero).optimize() - graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).invert() - - graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).invert() - # separator = "." - - def tens_no_zero(): - return ( - pynutil.delete("0") + graph_digit - | get_ties_digit( - get_abs_path("data/numbers/digit.tsv"), get_abs_path("data/numbers/ties.tsv") - ).invert() - | graph_teen - | (graph_ties + pynutil.delete("0")) + self.tens = graph_tens.optimize() + + graph_two_digit_non_zero = pynini.union(graph_digit, graph_tens, (pynutil.delete("0") + graph_digit)) + if not deterministic: + graph_two_digit_non_zero |= pynutil.add_weight( + pynini.union(graph_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit)), -0.001 ) - def hundred_non_zero(): - return (graph_digit_no_one + insert_space | pynini.cross("1", "ein ")) + pynutil.insert("hundert") + ( - pynini.closure(insert_space + pynutil.insert(AND, weight=0.0001), 0, 1) + insert_space + tens_no_zero() - | pynutil.delete("00") - ) | pynutil.delete("0") + tens_no_zero() + self.two_digit_non_zero = graph_two_digit_non_zero.optimize() - def thousand(): - return (hundred_non_zero() + insert_space + pynutil.insert("tausend") | pynutil.delete("000")) + ( - insert_space + hundred_non_zero() | pynutil.delete("000") + graph_final_two_digit_non_zero = pynini.union(final_digit, graph_tens, (pynutil.delete("0") + final_digit)) + if not deterministic: + graph_final_two_digit_non_zero |= pynutil.add_weight( + pynini.union(final_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + final_digit)), -0.001 ) - optional_plural_quantity_en = pynini.closure(pynutil.insert("en", weight=-0.0001), 0, 1) - optional_plural_quantity_n = pynini.closure(pynutil.insert("n", weight=-0.0001), 0, 1) - graph_million = pynini.union( - hundred_non_zero() + insert_space + pynutil.insert("million") + optional_plural_quantity_en, - pynutil.delete("000"), - ) + self.final_two_digit_non_zero = graph_final_two_digit_non_zero.optimize() - graph_billion = pynini.union( - hundred_non_zero() + insert_space + pynutil.insert("milliarde") + optional_plural_quantity_n, - pynutil.delete("000"), - ) + # Three digit strings + graph_hundreds = hundreds + pynini.union(pynutil.delete("00"), graph_tens, (pynutil.delete("0") + final_digit)) + if not deterministic: + graph_hundreds |= pynutil.add_weight( + hundreds + + pynini.union( + pynutil.delete("00"), + (graph_tens | pynutil.insert(NEMO_SPACE) + graph_tens), + (pynini.cross("0", NEMO_SPACE) + final_digit), + ), + -0.001, + ) + + self.hundreds = graph_hundreds.optimize() - graph_trillion = pynini.union( - hundred_non_zero() + insert_space + pynutil.insert("billion") + optional_plural_quantity_en, - pynutil.delete("000"), + # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + + graph_hundreds_component_at_least_one_non_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + graph_digit ) - graph_quadrillion = pynini.union( - hundred_non_zero() + insert_space + pynutil.insert("billiarde") + optional_plural_quantity_n, - pynutil.delete("000"), + graph_hundreds_component_at_least_one_non_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + digits_no_one + ) + self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( + graph_hundreds_component_at_least_one_non_zero_digit_no_one.optimize() ) - graph_quintillion = pynini.union( - hundred_non_zero() + insert_space + pynutil.insert("trillion") + optional_plural_quantity_en, - pynutil.delete("000"), + tusen = pynutil.insert("tusen") + etttusen = tusen + if not deterministic: + tusen |= pynutil.add_weight(pynutil.insert(" tusen"), -0.001) + etttusen = tusen + etttusen |= pynutil.add_weight(pynutil.insert("etttusen"), -0.001) + etttusen |= pynutil.add_weight(pynutil.insert(" etttusen"), -0.001) + etttusen |= pynutil.add_weight(pynutil.insert("ett tusen"), -0.001) + etttusen |= pynutil.add_weight(pynutil.insert(" ett tusen"), -0.001) + + following_hundred = insert_space + graph_hundreds_component_at_least_one_non_zero_digit + if not deterministic: + following_hundred |= graph_hundreds_component_at_least_one_non_zero_digit + + graph_thousands_component_at_least_one_non_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit, + graph_hundreds_component_at_least_one_non_zero_digit_no_one + + tusen + + (following_hundred | pynutil.delete("000")), + pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), + ) + self.graph_thousands_component_at_least_one_non_zero_digit = ( + graph_thousands_component_at_least_one_non_zero_digit.optimize() ) - graph_sextillion = pynini.union( - hundred_non_zero() + insert_space + pynutil.insert("trilliarde") + optional_plural_quantity_n, - pynutil.delete("000"), + graph_thousands_component_at_least_one_non_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit_no_one, + graph_hundreds_component_at_least_one_non_zero_digit_no_one + + tusen + + (following_hundred | pynutil.delete("000")), + pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), + ) + self.graph_thousands_component_at_least_one_non_zero_digit_no_one = ( + graph_thousands_component_at_least_one_non_zero_digit_no_one.optimize() ) - graph = pynini.union( - graph_sextillion - + insert_space - + graph_quintillion - + insert_space - + graph_quadrillion - + insert_space + + non_zero_no_one = graph_hundreds_component_at_least_one_non_zero_digit_no_one + graph_million = make_million("miljon", non_zero_no_one, deterministic) + graph_milliard = make_million("miljard", non_zero_no_one, deterministic) + graph_billion = make_million("biljon", non_zero_no_one, deterministic) + graph_billiard = make_million("biljard", non_zero_no_one, deterministic) + graph_trillion = make_million("triljon", non_zero_no_one, deterministic) + graph_trilliard = make_million("triljard", non_zero_no_one, deterministic) + + graph = ( + graph_trilliard + graph_trillion - + insert_space + + graph_billiard + graph_billion - + insert_space + + graph_milliard + graph_million - + insert_space - + thousand() - ) - - fix_syntax = [ - ("eins tausend", "ein tausend"), - ("eins millionen", "eine million"), - ("eins milliarden", "eine milliarde"), - ("eins billionen", "eine billion"), - ("eins billiarden", "eine billiarde"), - ] - fix_syntax = pynini.union(*[pynini.cross(*x) for x in fix_syntax]) + + (graph_thousands_component_at_least_one_non_zero_digit | pynutil.delete("000000")) + ) + self.graph = ( - ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) @ NEMO_DIGIT ** 24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) - @ pynini.cdrewrite(pynini.cross(" ", " "), "", "", NEMO_SIGMA) - @ pynini.cdrewrite(fix_syntax, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), SV_ALPHA, SV_ALPHA, NEMO_SIGMA + ) ) - self.graph |= graph_zero | pynini.cross("1", "eins") - - # self.graph = pynini.cdrewrite(pynutil.delete(separator), "", "", NEMO_SIGMA) @ self.graph - self.graph = self.graph.optimize() - - self.graph_hundred_component_at_least_one_none_zero_digit = ( - ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") - @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 3 - @ hundred_non_zero() - ) | pynini.cross("1", "eins") - self.graph_hundred_component_at_least_one_none_zero_digit = ( - self.graph_hundred_component_at_least_one_none_zero_digit.optimize() + self.graph_hundreds_component_at_least_one_non_zero_digit = ( + pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) + ) @ self.graph + self.graph_hundreds_component_at_least_one_non_zero_digit_en = ( + self.graph_hundreds_component_at_least_one_non_zero_digit + @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) ) + # For plurals, because the 'one' in 21, etc. still needs to agree + self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( + pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit, "input") - "1" + ) @ self.graph_hundreds_component_at_least_one_non_zero_digit + self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en = ( + pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit_en, "input") - "1" + ) @ self.graph_hundreds_component_at_least_one_non_zero_digit_en - self.two_digit_non_zero = ( - pynini.closure(NEMO_DIGIT, 1, 2) @ self.graph_hundred_component_at_least_one_none_zero_digit + zero_space = zero + insert_space + self.zero_space = zero_space + self.three_digits_read = pynini.union( + ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + zero_space + zero_space + digit, ) + self.three_digits_read_en = pynini.union( + ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, + zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + zero_space + zero_space + digit, + ) + self.three_digits_read_frac = pynini.union( + ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + zero_space + digit + insert_space + digit, + ) + self.three_digits_read_frac_en = pynini.union( + ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, + zero_space + digit + insert_space + digit, + ) + self.two_or_three_digits_read_frac = pynini.union( + ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, + zero_space + single_digits_graph + pynini.closure(insert_space + digit, 0, 1), + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), + zero_space + zero_space + zero, + single_digits_graph, + ) + self.two_or_three_digits_read_frac_en = pynini.union( + ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), + zero_space + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 0, 1), + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), + zero_space + zero_space + zero, + single_digits_graph, + ) + self.two_digits_read = pynini.union(((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, zero_space + digit) + self.two_digits_read_en = pynini.union( + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), + zero_space + digit, + ) + self.any_read_digit = ((NEMO_DIGIT - "0") @ digit) + pynini.closure(insert_space + digit) + if not deterministic: + self.three_digits_read |= pynutil.add_weight(digit + insert_space + digit + insert_space + digit, -0.001) + self.three_digits_read |= pynutil.add_weight( + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens + insert_space + digit, -0.001 + ) + self.three_digits_read |= pynutil.add_weight( + digit + insert_space + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, -0.001 + ) + self.two_digits_read |= pynutil.add_weight(digit + insert_space + digit, -0.001) + + self.graph |= zero + + self.graph_unfiltered = self.graph + self.graph = filter_punctuation(self.graph).optimize() + self.graph_en = self.graph @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) + self.graph_no_one = (pynini.project(self.graph, "input") - "1") @ self.graph + self.graph_no_one_en = (pynini.project(self.graph_en, "input") - "1") @ self.graph_en + + joiner_chars = pynini.union("-", "–", "—") + joiner = pynini.cross(joiner_chars, " till ") + self.range = self.graph + joiner + self.graph + if not deterministic: + either_one = self.graph | self.graph_en + self.range = either_one + joiner + either_one optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + if not deterministic: + final_graph |= pynutil.add_weight( + optional_minus_graph + pynutil.insert("integer: \"") + self.graph_en + pynutil.insert("\""), -0.001 + ) + final_graph |= pynutil.add_weight( + pynutil.insert("integer: \"") + self.single_digits_graph + pynutil.insert("\""), -0.001 + ) + final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() From 6b44d909df3a96d4af1192a6de38d895147664b3 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 4 Apr 2023 18:36:31 +0000 Subject: [PATCH 14/73] copy from sv Signed-off-by: Jim O'Regan --- .../text_normalization/pl/utils.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/utils.py diff --git a/nemo_text_processing/text_normalization/pl/utils.py b/nemo_text_processing/text_normalization/pl/utils.py new file mode 100644 index 000000000..6d20e29e0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels From 8d469929c2dd17b11aaf2b8c2d4d7994b8b19be9 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 4 Apr 2023 18:39:57 +0000 Subject: [PATCH 15/73] adapt from sv Signed-off-by: Jim O'Regan --- .../text_normalization/pl/graph_utils.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/graph_utils.py diff --git a/nemo_text_processing/text_normalization/pl/graph_utils.py b/nemo_text_processing/text_normalization/pl/graph_utils.py new file mode 100644 index 000000000..97d755f35 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/graph_utils.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space +from pynini.lib import byte, pynutil + +from .utils import get_abs_path, load_labels + +_ALPHA_UPPER = "AĄBCĆDEĘFGHIJKLŁMNŃOÓPQRSŚTUVWXYZŻŹ" +_ALPHA_LOWER = "aąbcćdeęfghijklłmnńoópqrsśtuvwxyzżź" + +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(_ALPHA_UPPER, _ALPHA_LOWER)]) +TO_UPPER = pynini.invert(TO_LOWER) + +PL_LOWER = pynini.union(*_ALPHA_LOWER).optimize() +PL_UPPER = pynini.union(*_ALPHA_UPPER).optimize() +PL_ALPHA = pynini.union(PL_LOWER, PL_UPPER).optimize() +PL_ALNUM = pynini.union(byte.DIGIT, PL_ALPHA).optimize() + +bos_or_space = pynini.union("[BOS]", " ") +eos_or_space = pynini.union("[EOS]", " ") + +ensure_space = pynini.cross(pynini.closure(delete_space, 0, 1), " ") + + +def roman_to_int(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000. + e.g. + "V" -> "5" + "i" -> "1" + + Args: + fst: Any fst. Composes fst onto Roman conversion outputs. + """ + + def _load_roman(file: str): + roman = load_labels(get_abs_path(file)) + roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman] + return pynini.string_map(roman_numerals) + + digit = _load_roman("data/roman/digit.tsv") + ties = _load_roman("data/roman/ties.tsv") + hundreds = _load_roman("data/roman/hundreds.tsv") + + graph = ( + digit + | ties + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + | ( + hundreds + + (ties | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + ).optimize() + + return graph @ fst From 387d3996369a97a80ee026224889b3a81e0bcfe6 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 4 Apr 2023 18:47:23 +0000 Subject: [PATCH 16/73] start adapting Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 45b826fab..18183d669 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -22,8 +22,8 @@ delete_space, insert_space, ) -from nemo_text_processing.text_normalization.sv.graph_utils import SV_ALPHA -from nemo_text_processing.text_normalization.sv.utils import get_abs_path +from nemo_text_processing.text_normalization.pl.graph_utils import PL_ALPHA +from nemo_text_processing.text_normalization.pl.utils import get_abs_path from pynini.lib import pynutil @@ -38,20 +38,10 @@ def make_million(number: str, non_zero_no_one: 'pynini.FstLike', deterministic: Returns: graph: A pynini.FstLike object """ - old_orth = number.replace("lj", "lli") graph = pynutil.add_weight(pynini.cross("001", number), -0.001) if not deterministic: - graph |= pynutil.add_weight(pynini.cross("001", old_orth), -0.001) - # 'ett' is usually wrong for these numbers, but it occurs - for one in ["en", "ett"]: - graph |= pynutil.add_weight(pynini.cross("001", f"{one} {number}"), -0.001) - graph |= pynutil.add_weight(pynini.cross("001", f"{one} {old_orth}"), -0.001) - graph |= pynutil.add_weight(pynini.cross("001", f"{one}{number}"), -0.001) - graph |= pynutil.add_weight(pynini.cross("001", f"{one}{old_orth}"), -0.001) + graph |= pynutil.add_weight(pynini.cross("001", "jeden {number}"), -0.001) graph |= non_zero_no_one + pynutil.insert(f" {number}er") - if not deterministic: - graph |= pynutil.add_weight(non_zero_no_one + pynutil.insert(f" {old_orth}er"), -0.001) - graph |= pynutil.add_weight(non_zero_no_one + pynutil.insert(f"{old_orth}er"), -0.001) graph |= pynutil.delete("000") graph += insert_space return graph @@ -89,8 +79,8 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. - "1000" -> cardinal { integer: "tusen" } - "2 000 000" -> cardinal { integer: "två miljon" } + "1000" -> cardinal { integer: "tysiąc" } + "2 000 000" -> cardinal { integer: "dwa miliony" } Args: deterministic: if True will provide a single transduction option, @@ -186,13 +176,6 @@ def __init__(self, deterministic: bool = True): tusen = pynutil.insert("tusen") etttusen = tusen - if not deterministic: - tusen |= pynutil.add_weight(pynutil.insert(" tusen"), -0.001) - etttusen = tusen - etttusen |= pynutil.add_weight(pynutil.insert("etttusen"), -0.001) - etttusen |= pynutil.add_weight(pynutil.insert(" etttusen"), -0.001) - etttusen |= pynutil.add_weight(pynutil.insert("ett tusen"), -0.001) - etttusen |= pynutil.add_weight(pynutil.insert(" ett tusen"), -0.001) following_hundred = insert_space + graph_hundreds_component_at_least_one_non_zero_digit if not deterministic: @@ -221,12 +204,12 @@ def __init__(self, deterministic: bool = True): ) non_zero_no_one = graph_hundreds_component_at_least_one_non_zero_digit_no_one - graph_million = make_million("miljon", non_zero_no_one, deterministic) - graph_milliard = make_million("miljard", non_zero_no_one, deterministic) - graph_billion = make_million("biljon", non_zero_no_one, deterministic) - graph_billiard = make_million("biljard", non_zero_no_one, deterministic) - graph_trillion = make_million("triljon", non_zero_no_one, deterministic) - graph_trilliard = make_million("triljard", non_zero_no_one, deterministic) + graph_million = make_million("milion", non_zero_no_one, deterministic) + graph_milliard = make_million("miliard", non_zero_no_one, deterministic) + graph_billion = make_million("bilion", non_zero_no_one, deterministic) + graph_billiard = make_million("biliard", non_zero_no_one, deterministic) + graph_trillion = make_million("trilion", non_zero_no_one, deterministic) + graph_trilliard = make_million("triliard", non_zero_no_one, deterministic) graph = ( graph_trilliard From eb70018f24120713eac174118b539f6f35abe5ad Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 4 Apr 2023 19:07:44 +0000 Subject: [PATCH 17/73] add __init__.py Signed-off-by: Jim O'Regan --- .../text_normalization/pl/__init__.py | 13 +++++++++++++ .../text_normalization/pl/data/numbers/__init__.py | 13 +++++++++++++ .../text_normalization/pl/data/ordinal/__init__.py | 13 +++++++++++++ .../text_normalization/pl/taggers/__init__.py | 13 +++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/__init__.py diff --git a/nemo_text_processing/text_normalization/pl/__init__.py b/nemo_text_processing/text_normalization/pl/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/taggers/__init__.py b/nemo_text_processing/text_normalization/pl/taggers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 104a3e518847423bf8bacd35ad6d769f7c0707ad Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 08:44:44 +0000 Subject: [PATCH 18/73] million/milliard inflections Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 18183d669..189c55767 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -27,7 +27,7 @@ from pynini.lib import pynutil -def make_million(number: str, non_zero_no_one: 'pynini.FstLike', deterministic: bool = True) -> 'pynini.FstLike': +def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'pynini.FstLike', case: str = None, deterministic: bool = True) -> 'pynini.FstLike': """ Helper function for millions/milliards and higher Args: @@ -38,10 +38,41 @@ def make_million(number: str, non_zero_no_one: 'pynini.FstLike', deterministic: Returns: graph: A pynini.FstLike object """ - graph = pynutil.add_weight(pynini.cross("001", number), -0.001) + if case is None: + sg_end = "" + pl_end = "y" + quant_end = "ów" + one = "jeden" + else: + SG = { + "loc": "ie", + "ins": "em", + "dat": "owi", + "gen": "a", + } + PL = { + "loc": "ach", + "ins": "ami", + "dat": "om", + "gen": "ów", + } + ONE = { + "loc": "jednym", + "ins": "jednym", + "dat": "jednemu", + "gen": "jednego", + } + sg_end = SG[case] + pl_end = PL[case] + one = ONE[case] + quant_end = pl_end + if case == "loc" and number.endswith("ard"): + sg_end = "zie" + graph = pynutil.add_weight(pynini.cross("001", f"{number}{sg_end}"), -0.001) if not deterministic: - graph |= pynutil.add_weight(pynini.cross("001", "jeden {number}"), -0.001) - graph |= non_zero_no_one + pynutil.insert(f" {number}er") + graph |= pynutil.add_weight(pynini.cross("001", f"{one} {number}{sg_end}"), -0.001) + graph |= non_zero_pl + pynutil.insert(f" {number}{pl_end}") + graph |= non_zero_quant + pynutil.insert(f" {number}{quant_end}") graph |= pynutil.delete("000") graph += insert_space return graph From 727c5ab030b2797f621a1e34e1619fca430bb44e Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 09:11:20 +0000 Subject: [PATCH 19/73] docs Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 189c55767..215a24b77 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -32,7 +32,9 @@ def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'py Helper function for millions/milliards and higher Args: number: the string of the number - non_zero_no_one: An fst of digits excluding 0 and 1, to prefix to the number + non_zero_pl: An fst of digits excluding 0, 1, 5-9, to prefix to plural forms (nom/acc) + non_zero_quant: An fst of digits excluding 0 and 1-4, to prefix to the quantity forms (nom/acc) + case: the string of the case (if None, nominative/accusative is presumed) deterministic: if True, generate a deterministic fst Returns: From 04cb49409ae211fda41416d3e5a9d71e59485f44 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 12:09:08 +0000 Subject: [PATCH 20/73] hundreds only really differ in genitive and instrumental Signed-off-by: Jim O'Regan --- .../text_normalization/pl/data/numbers/hundreds_gen.tsv | 9 +++++++++ .../text_normalization/pl/data/numbers/hundreds_ins.tsv | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv new file mode 100644 index 000000000..84b1ab48e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_gen.tsv @@ -0,0 +1,9 @@ +stu 1 +dwustu 2 +trzystu 3 +czterystu 4 +pięciuset 5 +sześciuset 6 +siedmiuset 7 +ośmiuset 8 +dziewięciuset 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv new file mode 100644 index 000000000..fb9209677 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_ins.tsv @@ -0,0 +1,9 @@ +stoma 1 +dwustoma 2 +trzystoma 3 +czterystoma 4 +pięciuset 5 +sześciuset 6 +siedmiuset 7 +ośmiuset 8 +dziewięciuset 9 From 3e0db505af2eab379631ee1d7ca9b6bde3668066 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 12:15:45 +0000 Subject: [PATCH 21/73] also teens Signed-off-by: Jim O'Regan --- .../text_normalization/pl/data/numbers/teen_gen.tsv | 10 ++++++++++ .../text_normalization/pl/data/numbers/teen_ins.tsv | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv new file mode 100644 index 000000000..7845f6d55 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_gen.tsv @@ -0,0 +1,10 @@ +dziesięciu 10 +jedenastu 11 +dwunastu 12 +trzynastu 13 +czternastu 14 +piętnastu 15 +szesnastu 16 +siedemnastu 17 +osiemnastu 18 +dziewiętnastu 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv new file mode 100644 index 000000000..0a2d28adb --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_ins.tsv @@ -0,0 +1,10 @@ +dziesięcioma 10 +jedenastoma 11 +dwunastoma 12 +trzynastoma 13 +czternastoma 14 +piętnastoma 15 +szesnastoma 16 +siedemnastoma 17 +osiemnastoma 18 +dziewiętnastoma 19 From 23db7a36d85cc23a7f018f000bc7ce5d0cf2b0bc Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 12:20:11 +0000 Subject: [PATCH 22/73] also tens Signed-off-by: Jim O'Regan --- .../text_normalization/pl/data/numbers/tens_gen.tsv | 8 ++++++++ .../text_normalization/pl/data/numbers/tens_ins.tsv | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv new file mode 100644 index 000000000..4475ae872 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_gen.tsv @@ -0,0 +1,8 @@ +dwudziestu 2 +trzydziestu 3 +czterdziestu 4 +pięćdziesięciu 5 +sześćdziesięciu 6 +siedemdziesięciu 7 +osiemdziesięciu 8 +dziewięćdziesięciu 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv new file mode 100644 index 000000000..77952c803 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_ins.tsv @@ -0,0 +1,8 @@ +dwudziestoma 2 +trzydziestoma 3 +czterdziestoma 4 +pięćdziesięcioma 5 +sześćdziesięcioma 6 +siedemdziesięcioma 7 +osiemdziesięcioma 8 +dziewięćdziesięcioma 9 From 721548b1e87fa63ada4216ec9db052a2c3fb81e1 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 12:45:32 +0000 Subject: [PATCH 23/73] 2, fem Signed-off-by: Jim O'Regan --- .../text_normalization/pl/data/numbers/two_fem_ins.tsv | 1 + .../text_normalization/pl/data/numbers/two_fem_nom.tsv | 1 + 2 files changed, 2 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv new file mode 100644 index 000000000..00b03394f --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv @@ -0,0 +1 @@ +dwiema 2 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv new file mode 100644 index 000000000..f5b9b2e19 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv @@ -0,0 +1 @@ +dwie 2 From 1f54b8f8c0c64a4c13961b74cf74c28e162ea833 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 5 Apr 2023 14:17:04 +0000 Subject: [PATCH 24/73] thousand Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/cardinal.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 215a24b77..f1c412b53 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -29,7 +29,7 @@ def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'pynini.FstLike', case: str = None, deterministic: bool = True) -> 'pynini.FstLike': """ - Helper function for millions/milliards and higher + Helper function for thousands/millions/milliards and higher Args: number: the string of the number non_zero_pl: An fst of digits excluding 0, 1, 5-9, to prefix to plural forms (nom/acc) @@ -74,6 +74,9 @@ def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'py if not deterministic: graph |= pynutil.add_weight(pynini.cross("001", f"{one} {number}{sg_end}"), -0.001) graph |= non_zero_pl + pynutil.insert(f" {number}{pl_end}") + # hack for the stem change in tysiąc (1000) + if number == "tysiąc": + number == "tysięc" graph |= non_zero_quant + pynutil.insert(f" {number}{quant_end}") graph |= pynutil.delete("000") graph += insert_space @@ -128,6 +131,12 @@ def __init__(self, deterministic: bool = True): ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + plural_3digits = NEMO_DIGIT + (NEMO_DIGIT - "1") + pynini.union("2", "3", "4") + quantity_3digits = NEMO_DIGIT + pynini.union( + "1" + NEMO_DIGIT, + (NEMO_DIGIT - "1") + pynini.union("0", "5", "6", "7", "8", "9") + ) + # Any single digit graph_digit = digit digits_no_one = (NEMO_DIGIT - "1") @ graph_digit @@ -136,13 +145,6 @@ def __init__(self, deterministic: bool = True): single_digits_graph = graph_digit | zero self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) - # spoken this way, so useful for e2e ASR - alt_ties = ties @ pynini.cdrewrite(ties_alt_endings, "", "[EOS]", NEMO_SIGMA) - if not deterministic: - ties |= pynutil.add_weight(alt_ties, -0.001) - ties |= pynutil.add_weight(pynini.cross("4", "förtio"), -0.001) - ties |= pynutil.add_weight(pynini.cross("4", "förti"), -0.001) - # Any double digit graph_tens = teen graph_ties = ties From 43a16dbf693e42e5fddadfa95f100fb1b5bee579 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 12 Jun 2023 09:10:51 +0000 Subject: [PATCH 25/73] noun + prefix pieces Signed-off-by: Jim O'Regan --- .../text_normalization/pl/data/numbers/digit_noun.tsv | 9 +++++++++ .../pl/data/numbers/digit_prefix.tsv | 9 +++++++++ .../pl/data/numbers/digit_prefix_nondet.tsv | 4 ++++ .../text_normalization/pl/data/numbers/teen_noun.tsv | 10 ++++++++++ .../text_normalization/pl/data/numbers/teen_prefix.tsv | 10 ++++++++++ .../text_normalization/pl/data/numbers/tens_noun.tsv | 8 ++++++++ .../text_normalization/pl/data/numbers/tens_prefix.tsv | 8 ++++++++ 7 files changed, 58 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv new file mode 100644 index 000000000..dafa51f38 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_noun.tsv @@ -0,0 +1,9 @@ +jedynka 1 +dwójka 2 +trójka 3 +czwórka 4 +piątka 5 +szóstka 6 +siódemka 7 +ósemka 8 +dziewiątka 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv new file mode 100644 index 000000000..ae80ccc50 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix.tsv @@ -0,0 +1,9 @@ +jedno 1 +dwu 2 +trzy 3 +cztero 4 +pięcio 5 +sześcio 6 +siedmio 7 +ośmio 8 +dziewięcio 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv new file mode 100644 index 000000000..f1a25cd56 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_prefix_nondet.tsv @@ -0,0 +1,4 @@ +dwój 2 +trój 3 +czworo 4 +czwór 4 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv new file mode 100644 index 000000000..f9617fe63 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_noun.tsv @@ -0,0 +1,10 @@ +dziesiątka 10 +jedenastka 11 +dwunastka 12 +trzynastka 13 +czternastka 14 +piętnastka 15 +szesnastka 16 +siedemnastka 17 +osiemnastka 18 +dziewiętnastka 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv new file mode 100644 index 000000000..52972da1e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teen_prefix.tsv @@ -0,0 +1,10 @@ +dziesięcio 10 +jedenasto 11 +dwunasto 12 +trzynasto 13 +czternasto 14 +piętnasto 15 +szesnasto 16 +siedemnasto 17 +osiemnasto 18 +dziewiętnasto 19 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv new file mode 100644 index 000000000..87e568040 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_noun.tsv @@ -0,0 +1,8 @@ +dwudziestka 2 +trzydziestka 3 +czterdziestka 4 +pięćdziesiątka 5 +sześćdziesiątka 6 +siedemdziesiątka 7 +osiemdziesiątka 8 +dziewięćdziesiątka 9 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv new file mode 100644 index 000000000..2e77b4c19 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tens_prefix.tsv @@ -0,0 +1,8 @@ +dwudziesto 2 +trzydziesto 3 +czterdziesto 4 +pięćdziesięcio 5 +sześćdziesięcio 6 +siedemdziesięcio 7 +osiemdziesięcio 8 +dziewięćdziesięcio 9 From ed227a63a571ce1e109225b4d1fb5f53eb29a7ad Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 12 Jun 2023 09:42:50 +0000 Subject: [PATCH 26/73] noun declension (for numeral nouns) Signed-off-by: Jim O'Regan --- .../pl/data/grammar/noun_feminine_ka.tsv | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv new file mode 100644 index 000000000..ae4401d05 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv @@ -0,0 +1,14 @@ +sg_nom ka +sg_gen ki +sg_dat ce +sg_acc kę +sg_ins ką +sg_loc ce +sg_voc ko +pl_nom ki +pl_gen ek +pl_dat kom +pl_acc ki +pl_ins kami +pl_loc kach +pl_voc ki From ec2e369cb8cf3dfddbfbcceebfcb39fe1b3943f2 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 12 Jun 2023 09:43:49 +0000 Subject: [PATCH 27/73] delete line Signed-off-by: Jim O'Regan --- nemo_text_processing/text_normalization/pl/taggers/cardinal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index f1c412b53..6c2aa6723 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -111,7 +111,6 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': return cardinal_string @ fst - class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. From d3f2ef10983df1a62656c97580b53bad198d4098 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 12 Jun 2023 09:43:55 +0000 Subject: [PATCH 28/73] Revert "delete line" This reverts commit ec2e369cb8cf3dfddbfbcceebfcb39fe1b3943f2. --- nemo_text_processing/text_normalization/pl/taggers/cardinal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 6c2aa6723..f1c412b53 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -111,6 +111,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': return cardinal_string @ fst + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. From a3a5bdba2fe02e4bea0fdc4fb1f8236a8612b21a Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 11:35:44 +0000 Subject: [PATCH 29/73] assign Signed-off-by: Jim O'Regan --- nemo_text_processing/text_normalization/pl/taggers/cardinal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index f1c412b53..ef2fb03e7 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -76,7 +76,7 @@ def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'py graph |= non_zero_pl + pynutil.insert(f" {number}{pl_end}") # hack for the stem change in tysiąc (1000) if number == "tysiąc": - number == "tysięc" + number = "tysięc" graph |= non_zero_quant + pynutil.insert(f" {number}{quant_end}") graph |= pynutil.delete("000") graph += insert_space From ac1e20398e19c413617418c8a8ecde36a393c8d1 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 12:47:13 +0000 Subject: [PATCH 30/73] missing default Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index d827cd358..80a168e2b 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -14,13 +14,16 @@ import pynini from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +# from nemo_text_processing.text_normalization.pl.taggers.cardinal import cardinal_graph from pynini.lib import pynutil -def adjective_inflection(word: str): - def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): +def adjective_inflection(word: str, compound: str = "") -> dict: + def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): if stem_b == "": stem_b = stem + if compound == "": + compound = stem_b + "o" return { "mi_sg_nom": mi_sg, "mi_sg_gen": stem + "ego", @@ -33,7 +36,7 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): "mp_pl_nom": mp_pl, "pl_ins": stem + vowel + "mi", "pl_loc": stem + vowel + "ch", - "compound": stem_b + "o", + "compound": compound, } stem_b = "" if word.endswith("en"): @@ -67,7 +70,12 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b=""): mi_sg = word mp_pl = word[:-2] + "ci" vowel = "y" - return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b) + elif word.endswith("y"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-1] + "i" + vowel = "y" + return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) class OrdinalFst(GraphFst): @@ -75,6 +83,7 @@ class OrdinalFst(GraphFst): Finite state transducer for classifying cardinals, e.g. "2." -> ordinal { integer: "drugi" } } "2-gi" -> ordinal { integer: "drugi" } } + "123." -> ordinal { integer: "sto dwudziesty trzeci" } } Args: deterministic: if True will provide a single transduction option, From 9342c8aa43d02fe419db511c7ff779bf4eaf4e58 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 13:19:07 +0000 Subject: [PATCH 31/73] make case dict of forms Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 80a168e2b..efea51607 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -14,6 +14,7 @@ import pynini from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels # from nemo_text_processing.text_normalization.pl.taggers.cardinal import cardinal_graph from pynini.lib import pynutil @@ -78,6 +79,21 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) +def make_graph_dict(filepath): + output_graph = {} + + word_tsv = load_labels(get_abs_path(filepath)) + for word, target in word_tsv: + word_forms = adjective_inflection(word) + + for key in word_forms: + if key in output_graph: + output_graph[key] = pynini.cross(word_forms[key], target) + else: + output_graph[key] |= pynini.cross(word_forms[key], target) + return output_graph + + class OrdinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. @@ -93,6 +109,13 @@ class OrdinalFst(GraphFst): def __init__(self, deterministic=False): super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + digits_tsv = load_labels(get_abs_path("data/ordinal/digit.tsv")) + for digit in digits_tsv: + word, num = digit + word_forms = adjective_inflection(word) + + + self.graph = ( ( pynini.closure(NEMO_DIGIT | pynini.accep(".")) From db21b4fd4d8624b94286789949f5d0ef2f6039d2 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 13:19:29 +0000 Subject: [PATCH 32/73] invert Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index efea51607..acb4c3bba 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -79,18 +79,22 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) -def make_graph_dict(filepath): +def make_graph_dict(filepath, invert=True): output_graph = {} - word_tsv = load_labels(get_abs_path(filepath)) for word, target in word_tsv: word_forms = adjective_inflection(word) - for key in word_forms: - if key in output_graph: - output_graph[key] = pynini.cross(word_forms[key], target) + if invert: + a = target + b = word_forms[key] + else: + a = word_forms[key] + b = target + if key not in output_graph: + output_graph[key] = pynini.cross(a, b) else: - output_graph[key] |= pynini.cross(word_forms[key], target) + output_graph[key] |= pynini.cross(a, b) return output_graph From 7dc5849f180e27d585a859a153ff7b63455f67ad Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 13:55:28 +0000 Subject: [PATCH 33/73] 2 digits Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index acb4c3bba..87a4c3b0a 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -13,7 +13,7 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels # from nemo_text_processing.text_normalization.pl.taggers.cardinal import cardinal_graph from pynini.lib import pynutil @@ -113,12 +113,11 @@ class OrdinalFst(GraphFst): def __init__(self, deterministic=False): super().__init__(name="ordinal", kind="classify", deterministic=deterministic) - digits_tsv = load_labels(get_abs_path("data/ordinal/digit.tsv")) - for digit in digits_tsv: - word, num = digit - word_forms = adjective_inflection(word) - - + self.digits_all = make_graph_dict("data/ordinal/digit.tsv") + self.tens_all = make_graph_dict("data/ordinal/tens.tsv") + self.teens_all = make_graph_dict("data/ordinal/teens.tsv") + self.hundreds_all = make_graph_dict("data/ordinal/hundreds.tsv") + two_digit_all = self.make_two_digit() self.graph = ( ( @@ -130,3 +129,27 @@ def __init__(self, deterministic=False): final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() + + def make_two_digit(self): + two_digits = {} + for key in self.digits_all: + two_digits[key] = self.tens_all[key] + pynutil.delete('0') + two_digits[key] |= pynutil.delete('0') + self.digits_all[key] + two_digits[key] |= self.teens_all[key] + if key != "compound": + two_digits[key] |= self.tens_all[key] + insert_space + self.digits_all[key] + else: + two_digits[key] |= self.tens_all[key] + self.digits_all[key] + return two_digits + +def make_two_digit(): + two_digits = {} + for key in digits_all: + two_digits[key] = tens_all[key] + pynutil.delete('0') + two_digits[key] |= pynutil.delete('0') + digits_all[key] + two_digits[key] |= teens_all[key] + if key != "compound": + two_digits[key] |= tens_all[key] + insert_space + digits_all[key] + else: + two_digits[key] |= tens_all[key] + digits_all[key] + return two_digits From 083e02fc827b68082991bff76e32be723fe5e127 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 13:55:42 +0000 Subject: [PATCH 34/73] del repl code Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 87a4c3b0a..d31564fd2 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -141,15 +141,3 @@ def make_two_digit(self): else: two_digits[key] |= self.tens_all[key] + self.digits_all[key] return two_digits - -def make_two_digit(): - two_digits = {} - for key in digits_all: - two_digits[key] = tens_all[key] + pynutil.delete('0') - two_digits[key] |= pynutil.delete('0') + digits_all[key] - two_digits[key] |= teens_all[key] - if key != "compound": - two_digits[key] |= tens_all[key] + insert_space + digits_all[key] - else: - two_digits[key] |= tens_all[key] + digits_all[key] - return two_digits From e19122b061e5e688d273e61af10efb14158cec9a Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 14:40:20 +0000 Subject: [PATCH 35/73] complete adjective paradigm Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index d31564fd2..e2b2573a3 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -79,6 +79,54 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) +def complete_paradigm(partial, complete=False): + partial["mi_sg_acc"] = partial["mi_sg_nom"] + partial["mi_sg_loc"] = partial["mi_sg_ins"] + partial["mi_sg_voc"] = partial["mi_sg_nom"] + # ma.sg same as mi.sg, except acc = gen + partial["ma_sg_nom"] = partial["mi_sg_nom"] + partial["ma_sg_gen"] = partial["mi_sg_gen"] + partial["ma_sg_dat"] = partial["mi_sg_dat"] + partial["ma_sg_acc"] = partial["mi_sg_gen"] + partial["ma_sg_ins"] = partial["mi_sg_ins"] + partial["ma_sg_loc"] = partial["mi_sg_loc"] + partial["ma_sg_voc"] = partial["mi_sg_voc"] + # mp.sg same as ma.sg + partial["mp_sg_nom"] = partial["ma_sg_nom"] + partial["mp_sg_gen"] = partial["ma_sg_gen"] + partial["mp_sg_dat"] = partial["ma_sg_dat"] + partial["mp_sg_acc"] = partial["ma_sg_acc"] + partial["mp_sg_ins"] = partial["ma_sg_ins"] + partial["mp_sg_loc"] = partial["ma_sg_loc"] + partial["mp_sg_voc"] = partial["ma_sg_voc"] + # nt.sg same as mi.sg aside from nom/acc/voc + partial["nt_sg_gen"] = partial["mi_sg_gen"] + partial["nt_sg_dat"] = partial["mi_sg_dat"] + partial["nt_sg_acc"] = partial["nt_sg_nom"] + partial["nt_sg_ins"] = partial["mi_sg_ins"] + partial["nt_sg_loc"] = partial["mi_sg_loc"] + partial["nt_sg_voc"] = partial["nt_sg_nom"] + # f.sg + partial["f_sg_dat"] = partial["f_sg_gen"] + partial["f_sg_acc"] = partial["f_sg_ins"] + partial["f_sg_loc"] = partial["f_sg_gen"] + partial["f_sg_voc"] = partial["f_sg_nom"] + # plurals + partial["mp_pl_acc"] = partial["pl_loc"] + partial["mp_pl_voc"] = partial["mp_pl_nom"] + partial["pl_nom"] = partial["nt_sg_nom"] + partial["pl_gen"] = partial["pl_loc"] + partial["pl_dat"] = partial["mi_sg_ins"] + partial["pl_acc"] = partial["pl_nom"] + partial["pl_voc"] = partial["pl_nom"] + if complete: + for gender in ["mi", "ma", "mp", "nt", "f"]: + for case in ["nom", "gen", "dat", "acc", "ins", "loc", "voc"]: + key = f'{gender}_pl_{case}' + if key not in partial: + partial[key] = partial[f'pl_{case}'] + + def make_graph_dict(filepath, invert=True): output_graph = {} word_tsv = load_labels(get_abs_path(filepath)) From 1d0927b3d804bcd4dc698d2e2e712abdd9bfa2d2 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 14:44:14 +0000 Subject: [PATCH 36/73] add here Signed-off-by: Jim O'Regan --- nemo_text_processing/text_normalization/pl/taggers/ordinal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index e2b2573a3..215787a66 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -127,11 +127,13 @@ def complete_paradigm(partial, complete=False): partial[key] = partial[f'pl_{case}'] -def make_graph_dict(filepath, invert=True): +def make_graph_dict(filepath, invert=True, complete=False): output_graph = {} word_tsv = load_labels(get_abs_path(filepath)) for word, target in word_tsv: word_forms = adjective_inflection(word) + if complete: + complete_paradigm(word_forms, complete=True) for key in word_forms: if invert: a = target From d4bc7c93d9ad84d7259f432b2e8fee4500d12cb1 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 14:52:10 +0000 Subject: [PATCH 37/73] helper Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 215787a66..0777cca07 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -191,3 +191,11 @@ def make_two_digit(self): else: two_digits[key] |= self.tens_all[key] + self.digits_all[key] return two_digits + + def all_to_graph(self, graph_dict, deterministic=False): + output_graph = graph_dict["mi_sg_nom"] + if not deterministic: + for key in graph_dict: + if key != "mi_sg_nom": + output_graph |= graph_dict[key] + return output_graph.optimize() \ No newline at end of file From 8725a6d9d49310242f31cdd2f584b31178adb08b Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 14:54:53 +0000 Subject: [PATCH 38/73] use Signed-off-by: Jim O'Regan --- .../text_normalization/pl/taggers/ordinal.py | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 0777cca07..055e4449c 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -148,6 +148,15 @@ def make_graph_dict(filepath, invert=True, complete=False): return output_graph +def all_to_graph(graph_dict, deterministic=False): + output_graph = graph_dict["mi_sg_nom"] + if not deterministic: + for key in graph_dict: + if key != "mi_sg_nom": + output_graph |= graph_dict[key] + return output_graph.optimize() + + class OrdinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. @@ -169,16 +178,22 @@ def __init__(self, deterministic=False): self.hundreds_all = make_graph_dict("data/ordinal/hundreds.tsv") two_digit_all = self.make_two_digit() - self.graph = ( - ( - pynini.closure(NEMO_DIGIT | pynini.accep(".")) - + pynutil.delete(pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")) - ) - @ cardinal_graph - ).optimize() - final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") - final_graph = self.add_tokens(final_graph) - self.fst = final_graph.optimize() + digits_graph = all_to_graph(self.digits_all, deterministic=deterministic) + tens_graph = all_to_graph(self.tens_all, deterministic=deterministic) + teens_graph = all_to_graph(self.teens_all, deterministic=deterministic) + hundreds_graph = all_to_graph(self.hundreds_all, deterministic=deterministic) + two_digit_graph = all_to_graph(two_digit_all, deterministic=deterministic) + + # self.graph = ( + # ( + # pynini.closure(NEMO_DIGIT | pynini.accep(".")) + # + pynutil.delete(pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")) + # ) + # @ cardinal_graph + # ).optimize() + # final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + # final_graph = self.add_tokens(final_graph) + # self.fst = final_graph.optimize() def make_two_digit(self): two_digits = {} @@ -192,10 +207,3 @@ def make_two_digit(self): two_digits[key] |= self.tens_all[key] + self.digits_all[key] return two_digits - def all_to_graph(self, graph_dict, deterministic=False): - output_graph = graph_dict["mi_sg_nom"] - if not deterministic: - for key in graph_dict: - if key != "mi_sg_nom": - output_graph |= graph_dict[key] - return output_graph.optimize() \ No newline at end of file From dbf297746f787b4d80b9d62f2fdfdcf6dae12770 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 17:27:52 +0000 Subject: [PATCH 39/73] add --- .../pl/data/grammar/noun_feminine_la.tsv | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_la.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_la.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_la.tsv new file mode 100644 index 000000000..0d04494cd --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_la.tsv @@ -0,0 +1,14 @@ +sg_nom la +sg_gen li +sg_dat li +sg_acc lę +sg_ins lą +sg_loc li +sg_voc lo +pl_nom le +pl_gen l +pl_dat lom +pl_acc li +pl_ins lami +pl_loc lach +pl_voc le From 6833e55f1f1bfc584d360aa06d26fd511d525511 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 17:39:16 +0000 Subject: [PATCH 40/73] add more Signed-off-by: Jim O'Regan --- .../pl/data/grammar/noun_feminine_ga.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_feminine_ia.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_feminine_ka.tsv | 2 +- .../pl/data/grammar/noun_feminine_na.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_feminine_ta.tsv | 14 ++++++++++++++ 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ga.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ia.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_na.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ta.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ga.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ga.tsv new file mode 100644 index 000000000..a38d46bed --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ga.tsv @@ -0,0 +1,14 @@ +sg_nom ga +sg_gen gi +sg_dat dze +sg_acc gę +sg_ins gą +sg_loc dze +sg_voc go +pl_nom gi +pl_gen g +pl_dat gom +pl_acc gi +pl_ins gami +pl_loc gach +pl_voc gi diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ia.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ia.tsv new file mode 100644 index 000000000..e07da049e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ia.tsv @@ -0,0 +1,14 @@ +sg_nom ia +sg_gen ii +sg_dat ii +sg_acc ię +sg_ins ią +sg_loc ii +sg_voc io +pl_nom ie +pl_gen ii +pl_dat iom +pl_acc ie +pl_ins iami +pl_loc iach +pl_voc ie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv index ae4401d05..2f53858c2 100644 --- a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv @@ -11,4 +11,4 @@ pl_dat kom pl_acc ki pl_ins kami pl_loc kach -pl_voc ki +pl_voc ki diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_na.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_na.tsv new file mode 100644 index 000000000..c32ae067b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_na.tsv @@ -0,0 +1,14 @@ +sg_nom na +sg_gen ny +sg_dat nie +sg_acc nę +sg_ins ną +sg_loc nie +sg_voc no +pl_nom ny +pl_gen n +pl_dat nom +pl_acc ny +pl_ins nami +pl_loc nach +pl_voc ny \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ta.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ta.tsv new file mode 100644 index 000000000..33823b746 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ta.tsv @@ -0,0 +1,14 @@ +sg_nom ta +sg_gen ty +sg_dat cie +sg_acc tę +sg_ins tą +sg_loc cie +sg_voc to +pl_nom ty +pl_gen t +pl_dat tom +pl_acc ty +pl_ins tami +pl_loc tach +pl_voc ty From 013c3d5961bf70a283b8b2ccdbfb6ec11c437931 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 18:14:37 +0000 Subject: [PATCH 41/73] add more Signed-off-by: Jim O'Regan --- .../pl/data/grammar/noun_masculine_ek.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_masculine_i.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_masculine_j.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_masculine_rz.tsv | 14 ++++++++++++++ .../pl/data/grammar/noun_masculine_y.tsv | 14 ++++++++++++++ 5 files changed, 70 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_ek.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_i.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_rz.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_y.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_ek.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_ek.tsv new file mode 100644 index 000000000..369e379ac --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_ek.tsv @@ -0,0 +1,14 @@ +sg_nom ek +sg_gen ka +sg_dat kowi +sg_acc ka +sg_ins kiem +sg_loc ku +sg_voc ku +pl_nom kowie +pl_gen ków +pl_dat kom +pl_acc ków +pl_ins kami +pl_loc kach +pl_voc kowie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_i.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_i.tsv new file mode 100644 index 000000000..66bae2274 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_i.tsv @@ -0,0 +1,14 @@ +sg_nom i +sg_gen ego +sg_dat emu +sg_acc ego +sg_ins im +sg_loc im +sg_voc i +pl_nom owie +pl_gen ich +pl_dat im +pl_acc ich +pl_ins imi +pl_loc ich +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv new file mode 100644 index 000000000..93302893e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc u +sg_voc u +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_rz.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_rz.tsv new file mode 100644 index 000000000..93302893e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_rz.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc u +sg_voc u +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_y.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_y.tsv new file mode 100644 index 000000000..88dc0860d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_y.tsv @@ -0,0 +1,14 @@ +sg_nom y +sg_gen ego +sg_dat emu +sg_acc ego +sg_ins ym +sg_loc ym +sg_voc y +pl_nom owie +pl_gen ych +pl_dat ym +pl_acc ych +pl_ins ymi +pl_loc ych +pl_voc owie From 9a67410b4a444e34c730a4921f1d466fa416e660 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 18:15:04 +0000 Subject: [PATCH 42/73] rm duplicate Signed-off-by: Jim O'Regan --- .../pl/data/grammar/noun_masculine_j.tsv | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv deleted file mode 100644 index 93302893e..000000000 --- a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_j.tsv +++ /dev/null @@ -1,14 +0,0 @@ -sg_nom -sg_gen a -sg_dat owi -sg_acc a -sg_ins em -sg_loc u -sg_voc u -pl_nom owie -pl_gen ów -pl_dat om -pl_acc ów -pl_ins ami -pl_loc ach -pl_voc owie From 4ed803f6c6c8bc1c145b49fee74667b07326fe76 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 18:16:08 +0000 Subject: [PATCH 43/73] rename Signed-off-by: Jim O'Regan --- .../pl/data/grammar/{noun_masculine_ek.tsv => noun_mp_ek.tsv} | 0 .../pl/data/grammar/{noun_masculine_i.tsv => noun_mp_i.tsv} | 0 .../pl/data/grammar/{noun_masculine_rz.tsv => noun_mp_rz.tsv} | 0 .../pl/data/grammar/{noun_masculine_y.tsv => noun_mp_y.tsv} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_masculine_ek.tsv => noun_mp_ek.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_masculine_i.tsv => noun_mp_i.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_masculine_rz.tsv => noun_mp_rz.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_masculine_y.tsv => noun_mp_y.tsv} (100%) diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_ek.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_ek.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_ek.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_ek.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_i.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_i.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_i.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_i.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_rz.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_rz.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_rz.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_rz.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_y.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_y.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_masculine_y.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_y.tsv From d820b39f8f8f28e11ceee03482e42f02c6df94fa Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 19:03:07 +0000 Subject: [PATCH 44/73] remove 1, it's special --- .../text_normalization/pl/data/numbers/digit.tsv | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv index 7efd57899..a8b198506 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv @@ -1,4 +1,3 @@ -jeden 1 dwa 2 trzy 3 cztery 4 From 492e5b1eda8941bd52f671175086926d0ff9d58d Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 19:12:09 +0000 Subject: [PATCH 45/73] mv --- .../pl/data/grammar/{noun_feminine_ga.tsv => noun_f_ga.tsv} | 0 .../pl/data/grammar/{noun_feminine_ia.tsv => noun_f_ia.tsv} | 0 .../pl/data/grammar/{noun_feminine_ka.tsv => noun_f_ka.tsv} | 0 .../pl/data/grammar/{noun_feminine_la.tsv => noun_f_la.tsv} | 0 .../pl/data/grammar/{noun_feminine_na.tsv => noun_f_na.tsv} | 0 .../pl/data/grammar/{noun_feminine_ta.tsv => noun_f_ta.tsv} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_feminine_ga.tsv => noun_f_ga.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_feminine_ia.tsv => noun_f_ia.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_feminine_ka.tsv => noun_f_ka.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_feminine_la.tsv => noun_f_la.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_feminine_na.tsv => noun_f_na.tsv} (100%) rename nemo_text_processing/text_normalization/pl/data/grammar/{noun_feminine_ta.tsv => noun_f_ta.tsv} (100%) diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ga.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ga.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ga.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ga.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ia.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ia.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ia.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ia.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ka.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ka.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ka.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_la.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_la.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_la.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_f_la.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_na.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_na.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_na.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_f_na.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ta.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ta.tsv similarity index 100% rename from nemo_text_processing/text_normalization/pl/data/grammar/noun_feminine_ta.tsv rename to nemo_text_processing/text_normalization/pl/data/grammar/noun_f_ta.tsv From 75f534cb370f9ea78ccf5f77069101a68730c676 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 19:12:22 +0000 Subject: [PATCH 46/73] ad --- .../pl/data/grammar/noun_mi_k.tsv | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv new file mode 100644 index 000000000..fd45b8826 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_k.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc +sg_ins iem +sg_loc u +sg_voc u +pl_nom i +pl_gen ów +pl_dat om +pl_acc i +pl_ins ami +pl_loc ach +pl_voc i From a80bfca979b9b96f36235458e3d809bb4c9bc534 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 19:14:47 +0000 Subject: [PATCH 47/73] copy --- .../text_normalization/pl/data/__init__.py | 13 +++++++++++++ .../text_normalization/pl/data/grammar/__init__.py | 13 +++++++++++++ .../text_normalization/pl/data/numbers/__init__.py | 2 +- .../text_normalization/pl/data/ordinal/__init__.py | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 nemo_text_processing/text_normalization/pl/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/__init__.py diff --git a/nemo_text_processing/text_normalization/pl/data/__init__.py b/nemo_text_processing/text_normalization/pl/data/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py b/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py index 6ebc808fa..341a77c5b 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py +++ b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py index 6ebc808fa..341a77c5b 100644 --- a/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 55db0fbec5031eb506291d0cbdc814b8de15c063 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 21:46:36 +0000 Subject: [PATCH 48/73] another helper --- .../text_normalization/pl/taggers/cardinal.py | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index ef2fb03e7..9dc1d88aa 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -23,7 +23,8 @@ insert_space, ) from nemo_text_processing.text_normalization.pl.graph_utils import PL_ALPHA -from nemo_text_processing.text_normalization.pl.utils import get_abs_path +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.pl.taggers.ordinal import adjective_inflection, complete_paradigm from pynini.lib import pynutil @@ -112,6 +113,27 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': return cardinal_string @ fst +def make_inflected_graph_dict(file_path: str, cross: str, deterministic=False) -> dict: + """ + Helper function to create a dictionary of pynini graphs from a TSV file. + Args: + file_path: Path to the TSV file containing the mappings. + cross: The string to cross with the second column of the TSV. + + Returns: + A dictionary where keys represent grammar and values are the corresponding pynini graphs. + """ + graph_dict = {} + for line in load_labels(get_abs_path(file_path)): + key, value = line[0], line[1] + if key not in graph_dict: + graph_dict[key] = pynini.cross(cross, value) + else: + if not deterministic: + graph_dict[key] |= pynini.cross(cross, value) + return graph_dict + + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. @@ -125,6 +147,18 @@ class CardinalFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + jeden_all = adjective_inflection("jeden") + jeden_graph = pynini.cross("1", jeden_all["mi_sg_nom"]) + if not deterministic: + for key in jeden_all: + if key == "mi_sg_nom": + continue + jeden_graph |= pynini.cross("1", jeden_all[key]) + complete_paradigm(jeden_all) + self.jeden_all = {a[0]: pynini.cross("1", a[1]) for a in jeden_all.items()} + + zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) @@ -264,7 +298,7 @@ def __init__(self, deterministic: bool = True): @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) @ pynini.cdrewrite( - pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), SV_ALPHA, SV_ALPHA, NEMO_SIGMA + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), PL_ALPHA, PL_ALPHA, NEMO_SIGMA ) ) From 10112d62277c5017871cdfc5cfb524930073f6bd Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Mon, 28 Jul 2025 21:47:40 +0000 Subject: [PATCH 49/73] dwa forms --- .../text_normalization/pl/data/numbers/digit.tsv | 1 - .../pl/data/numbers/dwa_forms.tsv | 13 +++++++++++++ .../pl/data/numbers/two_fem_ins.tsv | 1 - .../pl/data/numbers/two_fem_nom.tsv | 1 - 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv delete mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv delete mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv index a8b198506..840057285 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit.tsv @@ -1,4 +1,3 @@ -dwa 2 trzy 3 cztery 4 pięć 5 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv new file mode 100644 index 000000000..d5532f424 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/dwa_forms.tsv @@ -0,0 +1,13 @@ +mi_pl_nom dwa +mp_pl_nom dwaj +f_pl_nom dwie +pl_gen dwóch +pl_gen dwu +f_pl_ins dwiema +f_pl_ins dwoma +f_pl_ins dwu +mi_pl_ins dwoma +mi_pl_ins dwu +pl_dat dwóm +pl_dat dwom +pl_dat dwu diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv deleted file mode 100644 index 00b03394f..000000000 --- a/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_ins.tsv +++ /dev/null @@ -1 +0,0 @@ -dwiema 2 diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv deleted file mode 100644 index f5b9b2e19..000000000 --- a/nemo_text_processing/text_normalization/pl/data/numbers/two_fem_nom.tsv +++ /dev/null @@ -1 +0,0 @@ -dwie 2 From 32101f48a79fb3581f384f0f7be428430416ffe5 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 29 Jul 2025 19:09:08 +0000 Subject: [PATCH 50/73] inflect noun --- .../text_normalization/pl/taggers/cardinal.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 9dc1d88aa..d383877f0 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -114,15 +114,6 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': def make_inflected_graph_dict(file_path: str, cross: str, deterministic=False) -> dict: - """ - Helper function to create a dictionary of pynini graphs from a TSV file. - Args: - file_path: Path to the TSV file containing the mappings. - cross: The string to cross with the second column of the TSV. - - Returns: - A dictionary where keys represent grammar and values are the corresponding pynini graphs. - """ graph_dict = {} for line in load_labels(get_abs_path(file_path)): key, value = line[0], line[1] @@ -134,6 +125,21 @@ def make_inflected_graph_dict(file_path: str, cross: str, deterministic=False) - return graph_dict +def get_nominal_inflections(inflection_file, noun_file): + output = {} + inflections = {a[0]: a[1] for a in load_labels(get_abs_path(inflection_file))} + digit_noun_tsv = load_labels(get_abs_path(noun_file)) + for digit_noun in digit_noun_tsv: + word = digit_noun[0] + digit = digit_noun[1] + lemma_ending = inflections["sg_nom"] + assert word.endswith(lemma_ending), f"Word {word} does not end with {lemma_ending}" + stem = word[:-len(lemma_ending)] + wordforms = {k: stem + v for k, v in inflections.items()} + output[digit] = wordforms + return output + + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. From aed7c3dec519a5cc63acf39133372c439a7f7ffc Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 29 Jul 2025 19:30:05 +0000 Subject: [PATCH 51/73] inflect noun -> graph --- .../text_normalization/pl/taggers/cardinal.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index d383877f0..940fefaf4 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -140,6 +140,18 @@ def get_nominal_inflections(inflection_file, noun_file): return output +def get_nominal_graph(inflection_file, noun_file) -> 'pynini.FstLike': + output = {} + input = get_nominal_inflections(inflection_file, noun_file) + for item in input: + for key in input[item]: + if not key in output: + output[key] = pynini.cross(item, input[item][key]) + else: + output[key] |= pynini.cross(item, input[item][key]) + return output + + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. From 5297f0069815f9f1b52f42d14c9d1fee11137017 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 29 Jul 2025 19:30:26 +0000 Subject: [PATCH 52/73] type --- nemo_text_processing/text_normalization/pl/taggers/cardinal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 940fefaf4..c825fb422 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -140,7 +140,7 @@ def get_nominal_inflections(inflection_file, noun_file): return output -def get_nominal_graph(inflection_file, noun_file) -> 'pynini.FstLike': +def get_nominal_graph(inflection_file, noun_file): output = {} input = get_nominal_inflections(inflection_file, noun_file) for item in input: From bd78f1741cc157dccd22aa1ede26fe0774d8d9e5 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 29 Jul 2025 23:32:18 +0000 Subject: [PATCH 53/73] pieces --- nemo_text_processing/text_normalization/pl/taggers/ordinal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 055e4449c..7fbae22c5 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -184,6 +184,9 @@ def __init__(self, deterministic=False): hundreds_graph = all_to_graph(self.hundreds_all, deterministic=deterministic) two_digit_graph = all_to_graph(two_digit_all, deterministic=deterministic) + wieczny_forms = adjective_inflection("wieczny") + wieczny_acceptor = pynini.union(*wieczny_forms.values()) + # self.graph = ( # ( # pynini.closure(NEMO_DIGIT | pynini.accep(".")) From 3531d8a4d3e7aa75a2cf33dcbb29e36c4b68d837 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 29 Jul 2025 23:32:27 +0000 Subject: [PATCH 54/73] pieces --- .../text_normalization/pl/taggers/cardinal.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index c825fb422..ca3250c45 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -168,6 +168,8 @@ def __init__(self, deterministic: bool = True): jeden_all = adjective_inflection("jeden") jeden_graph = pynini.cross("1", jeden_all["mi_sg_nom"]) + # in compound numbers, jeden does not inflect + jeden_only = pynini.cross("1", jeden_all["mi_sg_nom"]) if not deterministic: for key in jeden_all: if key == "mi_sg_nom": @@ -175,7 +177,12 @@ def __init__(self, deterministic: bool = True): jeden_graph |= pynini.cross("1", jeden_all[key]) complete_paradigm(jeden_all) self.jeden_all = {a[0]: pynini.cross("1", a[1]) for a in jeden_all.items()} + self.zero_all = get_nominal_graph("data/grammar/noun_nt_ro.tsv", "data/numbers/zero.tsv") + cases = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] + jeden_filt = {} + for case in cases: + jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) From 34889246639598002e7690c8d02c088d3e7da5a1 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 29 Jul 2025 23:54:22 +0000 Subject: [PATCH 55/73] add --- .../pl/data/grammar/noun_nt_oje.tsv | 7 +++++++ .../pl/data/grammar/noun_nt_ro.tsv | 14 ++++++++++++++ .../text_normalization/pl/data/numbers/tysiac.tsv | 14 ++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv new file mode 100644 index 000000000..24a67362e --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_oje.tsv @@ -0,0 +1,7 @@ +pl_nom e +pl_gen ga +pl_dat gu +pl_acc e +pl_ins giem +pl_loc gu +pl_voc e diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv new file mode 100644 index 000000000..aac94d8b8 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_nt_ro.tsv @@ -0,0 +1,14 @@ +sg_nom o +sg_gen a +sg_dat u +sg_acc o +sg_ins em +sg_loc ze +sg_voc o +pl_nom a +pl_gen +pl_dat om +pl_acc a +pl_ins ami +pl_loc ach +pl_voc a diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv new file mode 100644 index 000000000..d9d148ea6 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/tysiac.tsv @@ -0,0 +1,14 @@ +sg_nom tysiąc +sg_gen tysiąca +sg_dat tysiącowi +sg_acc tysiąc +sg_ins tysiącem +sg_loc tysiącu +sg_voc tysiącu +pl_nom tysiące +pl_gen tysięcy +pl_dat tysiącom +pl_acc tysiące +pl_ins tysiącami +pl_loc tysiącach +pl_voc tysiące From 89d7ce03b40f3c59a451ef3b2fd11f711f12b889 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 15:43:30 +0000 Subject: [PATCH 56/73] move function --- .../text_normalization/pl/taggers/ordinal.py | 60 +------------------ .../text_normalization/pl/utils.py | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+), 59 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 7fbae22c5..5cde45e19 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -18,65 +18,7 @@ # from nemo_text_processing.text_normalization.pl.taggers.cardinal import cardinal_graph from pynini.lib import pynutil - -def adjective_inflection(word: str, compound: str = "") -> dict: - def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): - if stem_b == "": - stem_b = stem - if compound == "": - compound = stem_b + "o" - return { - "mi_sg_nom": mi_sg, - "mi_sg_gen": stem + "ego", - "mi_sg_dat": stem + "emu", - "mi_sg_ins": stem + vowel + "m", - "nt_sg_nom": stem + "e", - "f_sg_nom": stem_b + "a", - "f_sg_gen": stem + "ej", - "f_sg_ins": stem_b + "ą", - "mp_pl_nom": mp_pl, - "pl_ins": stem + vowel + "mi", - "pl_loc": stem + vowel + "ch", - "compound": compound, - } - stem_b = "" - if word.endswith("en"): - stem = word[:-2] + "n" - mi_sg = word - mp_pl = stem + "i" - vowel = "y" - elif word[-2:] in ["ni", "ci"]: - stem = word - mi_sg = word - mp_pl = word - vowel = "" - elif word.endswith("szy"): - stem = word[:-1] - mi_sg = word - mp_pl = word[:-2] + "i" - vowel = "y" - elif word.endswith("gi"): - stem = word - stem_b = word[:-1] - mi_sg = word - mp_pl = word[:-2] + "dzy" - vowel = "" - elif word.endswith("sty"): - stem = word[:-1] - mi_sg = word - mp_pl = word[:-3] + "ści" - vowel = "y" - elif word.endswith("ty"): - stem = word[:-1] - mi_sg = word - mp_pl = word[:-2] + "ci" - vowel = "y" - elif word.endswith("y"): - stem = word[:-1] - mi_sg = word - mp_pl = word[:-1] + "i" - vowel = "y" - return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection def complete_paradigm(partial, complete=False): diff --git a/nemo_text_processing/text_normalization/pl/utils.py b/nemo_text_processing/text_normalization/pl/utils.py index 6d20e29e0..b02cfe4c8 100644 --- a/nemo_text_processing/text_normalization/pl/utils.py +++ b/nemo_text_processing/text_normalization/pl/utils.py @@ -40,3 +40,63 @@ def load_labels(abs_path): with open(abs_path, encoding="utf-8") as label_tsv: labels = list(csv.reader(label_tsv, delimiter="\t")) return labels + + +def adjective_inflection(word: str, compound: str = "") -> dict: + def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): + if stem_b == "": + stem_b = stem + if compound == "": + compound = stem_b + "o" + return { + "mi_sg_nom": mi_sg, + "mi_sg_gen": stem + "ego", + "mi_sg_dat": stem + "emu", + "mi_sg_ins": stem + vowel + "m", + "nt_sg_nom": stem + "e", + "f_sg_nom": stem_b + "a", + "f_sg_gen": stem + "ej", + "f_sg_ins": stem_b + "ą", + "mp_pl_nom": mp_pl, + "pl_ins": stem + vowel + "mi", + "pl_loc": stem + vowel + "ch", + "compound": compound, + } + stem_b = "" + if word.endswith("en"): + stem = word[:-2] + "n" + mi_sg = word + mp_pl = stem + "i" + vowel = "y" + elif word[-2:] in ["ni", "ci"]: + stem = word + mi_sg = word + mp_pl = word + vowel = "" + elif word.endswith("szy"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "i" + vowel = "y" + elif word.endswith("gi"): + stem = word + stem_b = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "dzy" + vowel = "" + elif word.endswith("sty"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-3] + "ści" + vowel = "y" + elif word.endswith("ty"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "ci" + vowel = "y" + elif word.endswith("y"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-1] + "i" + vowel = "y" + return fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b, compound) From 699d97b11db314046d77c8f04578222b1ad26467 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 15:44:05 +0000 Subject: [PATCH 57/73] comment --- .../text_normalization/pl/taggers/cardinal.py | 493 +++++++++--------- 1 file changed, 247 insertions(+), 246 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index ca3250c45..a09934781 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -23,8 +23,8 @@ insert_space, ) from nemo_text_processing.text_normalization.pl.graph_utils import PL_ALPHA -from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels -from nemo_text_processing.text_normalization.pl.taggers.ordinal import adjective_inflection, complete_paradigm +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels +from nemo_text_processing.text_normalization.pl.taggers.ordinal import complete_paradigm from pynini.lib import pynutil @@ -178,253 +178,254 @@ def __init__(self, deterministic: bool = True): complete_paradigm(jeden_all) self.jeden_all = {a[0]: pynini.cross("1", a[1]) for a in jeden_all.items()} self.zero_all = get_nominal_graph("data/grammar/noun_nt_ro.tsv", "data/numbers/zero.tsv") + self.zero_sg = {x.replace("sg_", ""): y for x, y in self.zero_all.items() if x.startswith("sg_")} cases = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] jeden_filt = {} for case in cases: jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] - zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) - digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) - teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) - ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) - hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) - - plural_3digits = NEMO_DIGIT + (NEMO_DIGIT - "1") + pynini.union("2", "3", "4") - quantity_3digits = NEMO_DIGIT + pynini.union( - "1" + NEMO_DIGIT, - (NEMO_DIGIT - "1") + pynini.union("0", "5", "6", "7", "8", "9") - ) - - # Any single digit - graph_digit = digit - digits_no_one = (NEMO_DIGIT - "1") @ graph_digit - self.digit = graph_digit - - single_digits_graph = graph_digit | zero - self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) - - # Any double digit - graph_tens = teen - graph_ties = ties - if deterministic: - graph_tens |= graph_ties + (pynutil.delete('0') | graph_digit) - else: - graph_tens |= pynutil.add_weight(pynini.cross("18", "aderton"), -0.001) - graph_tens |= pynutil.add_weight( - graph_ties + (pynutil.delete('0') | (graph_digit | insert_space + graph_digit)), -0.001 - ) - - hundreds = digits_no_one + pynutil.insert("hundra") - hundreds |= pynini.cross("1", "hundra") - if not deterministic: - hundreds |= pynutil.add_weight(pynini.cross("1", "etthundra"), -0.001) - hundreds |= pynutil.add_weight(digit + pynutil.insert(NEMO_SPACE) + pynutil.insert("hundra"), -0.001) - - self.tens = graph_tens.optimize() - - graph_two_digit_non_zero = pynini.union(graph_digit, graph_tens, (pynutil.delete("0") + graph_digit)) - if not deterministic: - graph_two_digit_non_zero |= pynutil.add_weight( - pynini.union(graph_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit)), -0.001 - ) - - self.two_digit_non_zero = graph_two_digit_non_zero.optimize() - - graph_final_two_digit_non_zero = pynini.union(final_digit, graph_tens, (pynutil.delete("0") + final_digit)) - if not deterministic: - graph_final_two_digit_non_zero |= pynutil.add_weight( - pynini.union(final_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + final_digit)), -0.001 - ) - - self.final_two_digit_non_zero = graph_final_two_digit_non_zero.optimize() - - # Three digit strings - graph_hundreds = hundreds + pynini.union(pynutil.delete("00"), graph_tens, (pynutil.delete("0") + final_digit)) - if not deterministic: - graph_hundreds |= pynutil.add_weight( - hundreds - + pynini.union( - pynutil.delete("00"), - (graph_tens | pynutil.insert(NEMO_SPACE) + graph_tens), - (pynini.cross("0", NEMO_SPACE) + final_digit), - ), - -0.001, - ) - - self.hundreds = graph_hundreds.optimize() - - # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) - graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) - - graph_hundreds_component_at_least_one_non_zero_digit = graph_hundreds_component | ( - pynutil.delete("00") + graph_digit - ) - - graph_hundreds_component_at_least_one_non_zero_digit_no_one = graph_hundreds_component | ( - pynutil.delete("00") + digits_no_one - ) - self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( - graph_hundreds_component_at_least_one_non_zero_digit_no_one.optimize() - ) - - tusen = pynutil.insert("tusen") - etttusen = tusen - - following_hundred = insert_space + graph_hundreds_component_at_least_one_non_zero_digit - if not deterministic: - following_hundred |= graph_hundreds_component_at_least_one_non_zero_digit - - graph_thousands_component_at_least_one_non_zero_digit = pynini.union( - pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit, - graph_hundreds_component_at_least_one_non_zero_digit_no_one - + tusen - + (following_hundred | pynutil.delete("000")), - pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), - ) - self.graph_thousands_component_at_least_one_non_zero_digit = ( - graph_thousands_component_at_least_one_non_zero_digit.optimize() - ) - - graph_thousands_component_at_least_one_non_zero_digit_no_one = pynini.union( - pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit_no_one, - graph_hundreds_component_at_least_one_non_zero_digit_no_one - + tusen - + (following_hundred | pynutil.delete("000")), - pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), - ) - self.graph_thousands_component_at_least_one_non_zero_digit_no_one = ( - graph_thousands_component_at_least_one_non_zero_digit_no_one.optimize() - ) - - non_zero_no_one = graph_hundreds_component_at_least_one_non_zero_digit_no_one - graph_million = make_million("milion", non_zero_no_one, deterministic) - graph_milliard = make_million("miliard", non_zero_no_one, deterministic) - graph_billion = make_million("bilion", non_zero_no_one, deterministic) - graph_billiard = make_million("biliard", non_zero_no_one, deterministic) - graph_trillion = make_million("trilion", non_zero_no_one, deterministic) - graph_trilliard = make_million("triliard", non_zero_no_one, deterministic) - - graph = ( - graph_trilliard - + graph_trillion - + graph_billiard - + graph_billion - + graph_milliard - + graph_million - + (graph_thousands_component_at_least_one_non_zero_digit | pynutil.delete("000000")) - ) - - self.graph = ( - ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) - @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 - @ graph - @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) - @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) - @ pynini.cdrewrite( - pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), PL_ALPHA, PL_ALPHA, NEMO_SIGMA - ) - ) - - self.graph_hundreds_component_at_least_one_non_zero_digit = ( - pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) - ) @ self.graph - self.graph_hundreds_component_at_least_one_non_zero_digit_en = ( - self.graph_hundreds_component_at_least_one_non_zero_digit - @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) - ) - # For plurals, because the 'one' in 21, etc. still needs to agree - self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( - pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit, "input") - "1" - ) @ self.graph_hundreds_component_at_least_one_non_zero_digit - self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en = ( - pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit_en, "input") - "1" - ) @ self.graph_hundreds_component_at_least_one_non_zero_digit_en - - zero_space = zero + insert_space - self.zero_space = zero_space - self.three_digits_read = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), - zero_space + zero_space + digit, - ) - self.three_digits_read_en = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), - zero_space + zero_space + digit, - ) - self.three_digits_read_frac = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - zero_space + digit + insert_space + digit, - ) - self.three_digits_read_frac_en = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - zero_space + digit + insert_space + digit, - ) - self.two_or_three_digits_read_frac = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, - zero_space + single_digits_graph + pynini.closure(insert_space + digit, 0, 1), - single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), - zero_space + zero_space + zero, - single_digits_graph, - ) - self.two_or_three_digits_read_frac_en = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), - zero_space + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 0, 1), - single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), - zero_space + zero_space + zero, - single_digits_graph, - ) - self.two_digits_read = pynini.union(((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, zero_space + digit) - self.two_digits_read_en = pynini.union( - ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), - zero_space + digit, - ) - self.any_read_digit = ((NEMO_DIGIT - "0") @ digit) + pynini.closure(insert_space + digit) - if not deterministic: - self.three_digits_read |= pynutil.add_weight(digit + insert_space + digit + insert_space + digit, -0.001) - self.three_digits_read |= pynutil.add_weight( - ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens + insert_space + digit, -0.001 - ) - self.three_digits_read |= pynutil.add_weight( - digit + insert_space + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, -0.001 - ) - self.two_digits_read |= pynutil.add_weight(digit + insert_space + digit, -0.001) - - self.graph |= zero - - self.graph_unfiltered = self.graph - self.graph = filter_punctuation(self.graph).optimize() - self.graph_en = self.graph @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) - self.graph_no_one = (pynini.project(self.graph, "input") - "1") @ self.graph - self.graph_no_one_en = (pynini.project(self.graph_en, "input") - "1") @ self.graph_en - - joiner_chars = pynini.union("-", "–", "—") - joiner = pynini.cross(joiner_chars, " till ") - self.range = self.graph + joiner + self.graph - if not deterministic: - either_one = self.graph | self.graph_en - self.range = either_one + joiner + either_one - - optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) - - final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") - if not deterministic: - final_graph |= pynutil.add_weight( - optional_minus_graph + pynutil.insert("integer: \"") + self.graph_en + pynutil.insert("\""), -0.001 - ) - final_graph |= pynutil.add_weight( - pynutil.insert("integer: \"") + self.single_digits_graph + pynutil.insert("\""), -0.001 - ) - - final_graph = self.add_tokens(final_graph) - self.fst = final_graph.optimize() + # zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + # digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + # teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) + # ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) + # hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) + + # plural_3digits = NEMO_DIGIT + (NEMO_DIGIT - "1") + pynini.union("2", "3", "4") + # quantity_3digits = NEMO_DIGIT + pynini.union( + # "1" + NEMO_DIGIT, + # (NEMO_DIGIT - "1") + pynini.union("0", "5", "6", "7", "8", "9") + # ) + + # # Any single digit + # graph_digit = digit + # digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + # self.digit = graph_digit + + # single_digits_graph = graph_digit | zero + # self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) + + # # Any double digit + # graph_tens = teen + # graph_ties = ties + # if deterministic: + # graph_tens |= graph_ties + (pynutil.delete('0') | graph_digit) + # else: + # graph_tens |= pynutil.add_weight(pynini.cross("18", "aderton"), -0.001) + # graph_tens |= pynutil.add_weight( + # graph_ties + (pynutil.delete('0') | (graph_digit | insert_space + graph_digit)), -0.001 + # ) + + # hundreds = digits_no_one + pynutil.insert("hundra") + # hundreds |= pynini.cross("1", "hundra") + # if not deterministic: + # hundreds |= pynutil.add_weight(pynini.cross("1", "etthundra"), -0.001) + # hundreds |= pynutil.add_weight(digit + pynutil.insert(NEMO_SPACE) + pynutil.insert("hundra"), -0.001) + + # self.tens = graph_tens.optimize() + + # graph_two_digit_non_zero = pynini.union(graph_digit, graph_tens, (pynutil.delete("0") + graph_digit)) + # if not deterministic: + # graph_two_digit_non_zero |= pynutil.add_weight( + # pynini.union(graph_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit)), -0.001 + # ) + + # self.two_digit_non_zero = graph_two_digit_non_zero.optimize() + + # graph_final_two_digit_non_zero = pynini.union(final_digit, graph_tens, (pynutil.delete("0") + final_digit)) + # if not deterministic: + # graph_final_two_digit_non_zero |= pynutil.add_weight( + # pynini.union(final_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + final_digit)), -0.001 + # ) + + # self.final_two_digit_non_zero = graph_final_two_digit_non_zero.optimize() + + # # Three digit strings + # graph_hundreds = hundreds + pynini.union(pynutil.delete("00"), graph_tens, (pynutil.delete("0") + final_digit)) + # if not deterministic: + # graph_hundreds |= pynutil.add_weight( + # hundreds + # + pynini.union( + # pynutil.delete("00"), + # (graph_tens | pynutil.insert(NEMO_SPACE) + graph_tens), + # (pynini.cross("0", NEMO_SPACE) + final_digit), + # ), + # -0.001, + # ) + + # self.hundreds = graph_hundreds.optimize() + + # # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) + # graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + + # graph_hundreds_component_at_least_one_non_zero_digit = graph_hundreds_component | ( + # pynutil.delete("00") + graph_digit + # ) + + # graph_hundreds_component_at_least_one_non_zero_digit_no_one = graph_hundreds_component | ( + # pynutil.delete("00") + digits_no_one + # ) + # self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( + # graph_hundreds_component_at_least_one_non_zero_digit_no_one.optimize() + # ) + + # tusen = pynutil.insert("tusen") + # etttusen = tusen + + # following_hundred = insert_space + graph_hundreds_component_at_least_one_non_zero_digit + # if not deterministic: + # following_hundred |= graph_hundreds_component_at_least_one_non_zero_digit + + # graph_thousands_component_at_least_one_non_zero_digit = pynini.union( + # pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit, + # graph_hundreds_component_at_least_one_non_zero_digit_no_one + # + tusen + # + (following_hundred | pynutil.delete("000")), + # pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), + # ) + # self.graph_thousands_component_at_least_one_non_zero_digit = ( + # graph_thousands_component_at_least_one_non_zero_digit.optimize() + # ) + + # graph_thousands_component_at_least_one_non_zero_digit_no_one = pynini.union( + # pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit_no_one, + # graph_hundreds_component_at_least_one_non_zero_digit_no_one + # + tusen + # + (following_hundred | pynutil.delete("000")), + # pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), + # ) + # self.graph_thousands_component_at_least_one_non_zero_digit_no_one = ( + # graph_thousands_component_at_least_one_non_zero_digit_no_one.optimize() + # ) + + # non_zero_no_one = graph_hundreds_component_at_least_one_non_zero_digit_no_one + # graph_million = make_million("milion", non_zero_no_one, deterministic) + # graph_milliard = make_million("miliard", non_zero_no_one, deterministic) + # graph_billion = make_million("bilion", non_zero_no_one, deterministic) + # graph_billiard = make_million("biliard", non_zero_no_one, deterministic) + # graph_trillion = make_million("trilion", non_zero_no_one, deterministic) + # graph_trilliard = make_million("triliard", non_zero_no_one, deterministic) + + # graph = ( + # graph_trilliard + # + graph_trillion + # + graph_billiard + # + graph_billion + # + graph_milliard + # + graph_million + # + (graph_thousands_component_at_least_one_non_zero_digit | pynutil.delete("000000")) + # ) + + # self.graph = ( + # ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + # @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + # @ NEMO_DIGIT ** 24 + # @ graph + # @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + # @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + # @ pynini.cdrewrite( + # pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), PL_ALPHA, PL_ALPHA, NEMO_SIGMA + # ) + # ) + + # self.graph_hundreds_component_at_least_one_non_zero_digit = ( + # pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) + # ) @ self.graph + # self.graph_hundreds_component_at_least_one_non_zero_digit_en = ( + # self.graph_hundreds_component_at_least_one_non_zero_digit + # @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) + # ) + # # For plurals, because the 'one' in 21, etc. still needs to agree + # self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( + # pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit, "input") - "1" + # ) @ self.graph_hundreds_component_at_least_one_non_zero_digit + # self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en = ( + # pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit_en, "input") - "1" + # ) @ self.graph_hundreds_component_at_least_one_non_zero_digit_en + + # zero_space = zero + insert_space + # self.zero_space = zero_space + # self.three_digits_read = pynini.union( + # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + # zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + # zero_space + zero_space + digit, + # ) + # self.three_digits_read_en = pynini.union( + # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, + # zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + # zero_space + zero_space + digit, + # ) + # self.three_digits_read_frac = pynini.union( + # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + # zero_space + digit + insert_space + digit, + # ) + # self.three_digits_read_frac_en = pynini.union( + # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, + # zero_space + digit + insert_space + digit, + # ) + # self.two_or_three_digits_read_frac = pynini.union( + # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, + # zero_space + single_digits_graph + pynini.closure(insert_space + digit, 0, 1), + # single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), + # zero_space + zero_space + zero, + # single_digits_graph, + # ) + # self.two_or_three_digits_read_frac_en = pynini.union( + # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, + # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), + # zero_space + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 0, 1), + # single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), + # zero_space + zero_space + zero, + # single_digits_graph, + # ) + # self.two_digits_read = pynini.union(((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, zero_space + digit) + # self.two_digits_read_en = pynini.union( + # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), + # zero_space + digit, + # ) + # self.any_read_digit = ((NEMO_DIGIT - "0") @ digit) + pynini.closure(insert_space + digit) + # if not deterministic: + # self.three_digits_read |= pynutil.add_weight(digit + insert_space + digit + insert_space + digit, -0.001) + # self.three_digits_read |= pynutil.add_weight( + # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens + insert_space + digit, -0.001 + # ) + # self.three_digits_read |= pynutil.add_weight( + # digit + insert_space + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, -0.001 + # ) + # self.two_digits_read |= pynutil.add_weight(digit + insert_space + digit, -0.001) + + # self.graph |= zero + + # self.graph_unfiltered = self.graph + # self.graph = filter_punctuation(self.graph).optimize() + # self.graph_en = self.graph @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) + # self.graph_no_one = (pynini.project(self.graph, "input") - "1") @ self.graph + # self.graph_no_one_en = (pynini.project(self.graph_en, "input") - "1") @ self.graph_en + + # joiner_chars = pynini.union("-", "–", "—") + # joiner = pynini.cross(joiner_chars, " till ") + # self.range = self.graph + joiner + self.graph + # if not deterministic: + # either_one = self.graph | self.graph_en + # self.range = either_one + joiner + either_one + + # optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + # final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + # if not deterministic: + # final_graph |= pynutil.add_weight( + # optional_minus_graph + pynutil.insert("integer: \"") + self.graph_en + pynutil.insert("\""), -0.001 + # ) + # final_graph |= pynutil.add_weight( + # pynutil.insert("integer: \"") + self.single_digits_graph + pynutil.insert("\""), -0.001 + # ) + + # final_graph = self.add_tokens(final_graph) + # self.fst = final_graph.optimize() From 015c3ea882b323d7fb8b52a33ce07087faac60f6 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 15:48:04 +0000 Subject: [PATCH 58/73] refactor --- .../text_normalization/pl/taggers/ordinal.py | 10 +--------- nemo_text_processing/text_normalization/pl/utils.py | 4 ++++ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 5cde45e19..ddaa16a03 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -14,6 +14,7 @@ import pynini from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.pl.graph_utils import all_to_graph from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels # from nemo_text_processing.text_normalization.pl.taggers.cardinal import cardinal_graph from pynini.lib import pynutil @@ -90,15 +91,6 @@ def make_graph_dict(filepath, invert=True, complete=False): return output_graph -def all_to_graph(graph_dict, deterministic=False): - output_graph = graph_dict["mi_sg_nom"] - if not deterministic: - for key in graph_dict: - if key != "mi_sg_nom": - output_graph |= graph_dict[key] - return output_graph.optimize() - - class OrdinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. diff --git a/nemo_text_processing/text_normalization/pl/utils.py b/nemo_text_processing/text_normalization/pl/utils.py index b02cfe4c8..a6d5f34dd 100644 --- a/nemo_text_processing/text_normalization/pl/utils.py +++ b/nemo_text_processing/text_normalization/pl/utils.py @@ -43,6 +43,10 @@ def load_labels(abs_path): def adjective_inflection(word: str, compound: str = "") -> dict: + """ + inflect adjectives based on their endings. + This includes things like ordinals and 'jeden' (1) which inflect like adjectives. + """ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): if stem_b == "": stem_b = stem From d43ec8cdd1604dd83db100fb2b5274f4eab99455 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 15:50:08 +0000 Subject: [PATCH 59/73] generalise a little --- .../text_normalization/pl/graph_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/graph_utils.py b/nemo_text_processing/text_normalization/pl/graph_utils.py index 97d755f35..4a296e5c2 100644 --- a/nemo_text_processing/text_normalization/pl/graph_utils.py +++ b/nemo_text_processing/text_normalization/pl/graph_utils.py @@ -67,3 +67,15 @@ def _load_roman(file: str): ).optimize() return graph @ fst + + +def all_to_graph(graph_dict, deterministic=False): + for default_key in ["mi_sg_nom", "sg_nom", "nom"]: + if default_key in graph_dict: + break + output_graph = graph_dict[default_key] + if not deterministic: + for key in graph_dict: + if key != default_key: + output_graph |= graph_dict[key] + return output_graph.optimize() From 8120b2f3bfdcd4e09416d87f9415039ea70ca6ec Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 16:35:47 +0000 Subject: [PATCH 60/73] so we can get forms of digits --- .../text_normalization/pl/taggers/cardinal.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index a09934781..f5228e539 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -152,6 +152,26 @@ def get_nominal_graph(inflection_file, noun_file): return output +def dict_to_graph(input_dict: dict, deterministic: bool = True) -> dict: + graph_dict = {} + for key, value in input_dict.items(): + if not key in graph_dict: + graph_dict[key] = {} + for subkey, subvalue in value.items(): + rest = [] + if type(subvalue) is list: + form = subvalue[0] + rest = subvalue[1:] + else: + form = subvalue + graph = pynini.cross(key, form) + if not deterministic and rest != []: + for alt in rest: + graph |= pynini.cross(key, alt) + graph_dict[key][subkey] = graph + return graph_dict + + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals, e.g. @@ -184,6 +204,8 @@ def __init__(self, deterministic: bool = True): jeden_filt = {} for case in cases: jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] + # something similar for dwa + # zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) # digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) From 5122542ac811706ee82a23f79094aa223388e88b Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 16:41:31 +0000 Subject: [PATCH 61/73] copilot --- .../text_normalization/pl/taggers/cardinal.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index f5228e539..b103015ea 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -153,22 +153,25 @@ def get_nominal_graph(inflection_file, noun_file): def dict_to_graph(input_dict: dict, deterministic: bool = True) -> dict: + """ + Converts a nested dictionary of forms to a dict of pynini.FSTs. + Example input: + {'2': {'mi_pl_ins': ['form1', 'form2'], 'mi_sg_nom': 'form3'}} + Output: + {'2': {'mi_pl_ins': FST, 'mi_sg_nom': FST}} + """ graph_dict = {} for key, value in input_dict.items(): - if not key in graph_dict: - graph_dict[key] = {} - for subkey, subvalue in value.items(): - rest = [] - if type(subvalue) is list: - form = subvalue[0] - rest = subvalue[1:] - else: - form = subvalue - graph = pynini.cross(key, form) - if not deterministic and rest != []: - for alt in rest: + graph_dict[key] = {} + for subkey, subvalue in value.items(): + if isinstance(subvalue, list): + graph = pynini.cross(key, subvalue[0]) + if not deterministic: + for alt in subvalue[1:]: graph |= pynini.cross(key, alt) - graph_dict[key][subkey] = graph + else: + graph = pynini.cross(key, subvalue) + graph_dict[key][subkey] = graph return graph_dict From 357c545c012087bf0e17cccd406d1acfea2cbe78 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 17:06:33 +0000 Subject: [PATCH 62/73] complete --- .../pl/data/numbers/digit_forms.tsv | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv new file mode 100644 index 000000000..bcafd4021 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv @@ -0,0 +1,32 @@ +2 mi_pl_nom dwa +2 mp_pl_nom dwaj +2 f_pl_nom dwie +2 pl_gen dwóch +2 pl_gen dwu +2 f_pl_ins dwiema +2 f_pl_ins dwoma +2 f_pl_ins dwu +2 mi_pl_ins dwoma +2 mi_pl_ins dwu +2 pl_dat dwóm +2 pl_dat dwom +2 pl_dat dwu +2 compound dwu +3 mi_pl_nom trzy +3 mp_pl_nom trzej +3 pl_gen trzech +3 mi_pl_ins trzema +3 pl_dat trzem +3 compound trzy +4 mi_pl_nom cztery +4 mp_pl_nom czterej +4 pl_gen czterech +4 mi_pl_ins czterema +4 pl_dat czterem +4 compound cztero +5 mi_pl_nom pięć +5 mp_pl_nom pięciu +5 pl_gen pięciu +5 mi_pl_ins pięcioma +5 pl_dat pięciu +5 compound pięcio \ No newline at end of file From 261b906bf0ece9bffc623440a4059bbcd8126c36 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 17:19:30 +0000 Subject: [PATCH 63/73] complete --- .../pl/data/numbers/teens_forms.tsv | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv new file mode 100644 index 000000000..aedd4ca9d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/teens_forms.tsv @@ -0,0 +1,60 @@ +10 mi_pl_nom dziesięć +10 mp_pl_nom dziesięciu +10 pl_gen dziesięciu +10 pl_ins dziesięcioma +10 pl_dat dziesięciu +10 compound dziesięcio +11 mi_pl_nom jedenaście +11 mp_pl_nom jedenastu +11 pl_gen jedenastu +11 pl_ins jedenastoma +11 pl_dat jedenastu +11 compound jedenasto +12 mi_pl_nom dwanaście +12 mp_pl_nom dwunastu +12 pl_gen dwunastu +12 pl_ins dwunastoma +12 pl_dat dwunastu +12 compound dwunasto +13 mi_pl_nom trzynaście +13 mp_pl_nom trzynastu +13 pl_gen trzynastu +13 pl_ins trzynastoma +13 pl_dat trzynastu +13 compound trzynasto +14 mi_pl_nom czternaście +14 mp_pl_nom czternastu +14 pl_gen czternastu +14 pl_ins czternastoma +14 pl_dat czternastu +14 compound czternasto +15 mi_pl_nom piętnaście +15 mp_pl_nom piętnastu +15 pl_gen piętnastu +15 pl_ins piętnastoma +15 pl_dat piętnastu +15 compound piętnasto +16 mi_pl_nom szesnaście +16 mp_pl_nom szesnastu +16 pl_gen szesnastu +16 pl_ins szesnastoma +16 pl_dat szesnastu +16 compound szesnasto +17 mi_pl_nom siedemnaście +17 mp_pl_nom siedemnastu +17 pl_gen siedemnastu +17 pl_ins siedemnastoma +17 pl_dat siedemnastu +17 compound siedemnasto +18 mi_pl_nom osiemnaście +18 mp_pl_nom osiemnastu +18 pl_gen osiemnastu +18 pl_ins osiemnastoma +18 pl_dat osiemnastu +18 compound osiemnasto +19 mi_pl_nom dziewiętnaście +19 mp_pl_nom dziewiętnastu +19 pl_gen dziewiętnastu +19 pl_ins dziewiętnastoma +19 pl_dat dziewiętnastu +19 compound dziewiętnasto From f6c10c252725ed8754903f7dd53325f063fd99a1 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 18:25:07 +0000 Subject: [PATCH 64/73] complete --- .../pl/data/numbers/digit_forms.tsv | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv index bcafd4021..6bcc308e7 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv +++ b/nemo_text_processing/text_normalization/pl/data/numbers/digit_forms.tsv @@ -15,18 +15,42 @@ 3 mi_pl_nom trzy 3 mp_pl_nom trzej 3 pl_gen trzech -3 mi_pl_ins trzema +3 pl_ins trzema 3 pl_dat trzem 3 compound trzy 4 mi_pl_nom cztery 4 mp_pl_nom czterej 4 pl_gen czterech -4 mi_pl_ins czterema +4 pl_ins czterema 4 pl_dat czterem 4 compound cztero 5 mi_pl_nom pięć 5 mp_pl_nom pięciu 5 pl_gen pięciu -5 mi_pl_ins pięcioma +5 pl_ins pięcioma 5 pl_dat pięciu -5 compound pięcio \ No newline at end of file +5 compound pięcio +6 mi_pl_nom sześć +6 mp_pl_nom sześciu +6 pl_gen sześciu +6 pl_ins sześcioma +6 pl_dat sześciu +6 compound sześcio +7 mi_pl_nom siedem +7 mp_pl_nom siedmiu +7 pl_gen siedmiu +7 pl_ins siedmioma +7 pl_dat siedmiu +7 compound siedmio +8 mi_pl_nom osiem +8 mp_pl_nom ośmiu +8 pl_gen ośmiu +8 pl_ins ośmioma +8 pl_dat ośmiu +8 compound ośmio +9 mi_pl_nom dziewięć +9 mp_pl_nom dziewięciu +9 pl_gen dziewięciu +9 pl_ins dziewięcioma +9 pl_dat dziewięciu +9 compound dziewięcio From c0a3283467b9c07563de4dc499f64b62acd8d738 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 18:42:59 +0000 Subject: [PATCH 65/73] try --- .../text_normalization/pl/taggers/cardinal.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index b103015ea..25102ca22 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -152,6 +152,25 @@ def get_nominal_graph(inflection_file, noun_file): return output +def get_digit_forms(filepath): + """ + Returns a dictionary of digit forms for Polish numbers. + """ + output = {} + for line in load_labels(get_abs_path(filepath)): + digit, grammar, form = line[0], line[1], line[2] + if not digit in output: + output[digit] = {} + if grammar not in output[digit]: + output[digit][grammar] = form + else: + if type(output[digit][grammar]) is list: + output[digit][grammar].append(form) + else: + output[digit][grammar] = [output[digit][grammar], form] + return output + + def dict_to_graph(input_dict: dict, deterministic: bool = True) -> dict: """ Converts a nested dictionary of forms to a dict of pynini.FSTs. @@ -204,10 +223,25 @@ def __init__(self, deterministic: bool = True): self.zero_sg = {x.replace("sg_", ""): y for x, y in self.zero_all.items() if x.startswith("sg_")} cases = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] + dwa_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "mi_pl_ins", "pl_gen", "mi_pl_nom"] + pl_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] + qnt_cases = ["mi_pl_nom", "pl_gen", "pl_gen", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] jeden_filt = {} for case in cases: jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] # something similar for dwa + digit_forms_all = get_digit_forms("data/numbers/digit_forms.tsv") + digit_graph = dict_to_graph(digit_forms_all, deterministic=deterministic) + digit_pl = {} + for case, trg in zip(cases, dwa_cases): + digit_pl[case] = digit_graph["2"][trg] + for pl_digit in ["3", "4"]: + for case, trg in zip(cases, pl_cases): + digit_pl[case] |= digit_graph[pl_digit][trg] + + # one does not inflect in compound numbers, so we use the nominative form + # e.g., https://www.poradnia-jezykowa.uni.lodz.pl/szczegoly/jeden-w-liczebnikach-wielowyrazowych + # zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) From e99d7280c00e1d940d707fcdea89285aa180d107 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 18:45:17 +0000 Subject: [PATCH 66/73] try --- .../text_normalization/pl/taggers/cardinal.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 25102ca22..056d47064 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -233,11 +233,12 @@ def __init__(self, deterministic: bool = True): digit_forms_all = get_digit_forms("data/numbers/digit_forms.tsv") digit_graph = dict_to_graph(digit_forms_all, deterministic=deterministic) digit_pl = {} - for case, trg in zip(cases, dwa_cases): - digit_pl[case] = digit_graph["2"][trg] - for pl_digit in ["3", "4"]: - for case, trg in zip(cases, pl_cases): - digit_pl[case] |= digit_graph[pl_digit][trg] + for idx in range(len(cases)): + digit_pl[cases[idx]] = pynini.union( + digit_graph["2"][dwa_cases[idx]], + digit_graph["3"][pl_cases[idx]], + digit_graph["4"][pl_cases[idx]], + ).optimize() # one does not inflect in compound numbers, so we use the nominative form # e.g., https://www.poradnia-jezykowa.uni.lodz.pl/szczegoly/jeden-w-liczebnikach-wielowyrazowych From 79a991f6824707ea0e45d7c65d83ae26ac1f833d Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 19:02:08 +0000 Subject: [PATCH 67/73] try --- .../text_normalization/pl/taggers/cardinal.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 056d47064..705664aab 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -28,6 +28,9 @@ from pynini.lib import pynutil +CASES = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] + + def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'pynini.FstLike', case: str = None, deterministic: bool = True) -> 'pynini.FstLike': """ Helper function for thousands/millions/milliards and higher @@ -222,22 +225,21 @@ def __init__(self, deterministic: bool = True): self.zero_all = get_nominal_graph("data/grammar/noun_nt_ro.tsv", "data/numbers/zero.tsv") self.zero_sg = {x.replace("sg_", ""): y for x, y in self.zero_all.items() if x.startswith("sg_")} - cases = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] dwa_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "mi_pl_ins", "pl_gen", "mi_pl_nom"] pl_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] qnt_cases = ["mi_pl_nom", "pl_gen", "pl_gen", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] jeden_filt = {} - for case in cases: + for case in CASES: jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] # something similar for dwa digit_forms_all = get_digit_forms("data/numbers/digit_forms.tsv") digit_graph = dict_to_graph(digit_forms_all, deterministic=deterministic) digit_pl = {} - for idx in range(len(cases)): - digit_pl[cases[idx]] = pynini.union( + for idx in range(len(CASES)): + digit_pl[CASES[idx]] = pynini.union( digit_graph["2"][dwa_cases[idx]], digit_graph["3"][pl_cases[idx]], - digit_graph["4"][pl_cases[idx]], + digit_graph["4"][pl_cases[idx]] ).optimize() # one does not inflect in compound numbers, so we use the nominative form From faea1d0b11df81a98657c555cc18aea9af0f0101 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Wed, 30 Jul 2025 19:11:05 +0000 Subject: [PATCH 68/73] commnet --- .../text_normalization/pl/taggers/cardinal.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 705664aab..bc3cb7dc5 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -229,8 +229,16 @@ def __init__(self, deterministic: bool = True): pl_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] qnt_cases = ["mi_pl_nom", "pl_gen", "pl_gen", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] jeden_filt = {} + jeden_compound = {} + # jeden (one) does not inflect in compound numbers, so we use the nominative form + # e.g., https://www.poradnia-jezykowa.uni.lodz.pl/szczegoly/jeden-w-liczebnikach-wielowyrazowych + # but a lot of people get this wrong, so we also include the inflected forms + # This is different from Russian; also, jeden in compounds is a quantity, not singular for case in CASES: jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] + jeden_compound[case] = jeden_all[f'mi_sg_nom'] + if not deterministic: + jeden_compound[case] |= self.jeden_all[f'mi_sg_{case}'] # something similar for dwa digit_forms_all = get_digit_forms("data/numbers/digit_forms.tsv") digit_graph = dict_to_graph(digit_forms_all, deterministic=deterministic) @@ -242,8 +250,6 @@ def __init__(self, deterministic: bool = True): digit_graph["4"][pl_cases[idx]] ).optimize() - # one does not inflect in compound numbers, so we use the nominative form - # e.g., https://www.poradnia-jezykowa.uni.lodz.pl/szczegoly/jeden-w-liczebnikach-wielowyrazowych From 66db6c387108a3c16a18b043b64c3df0796a8df6 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 5 Aug 2025 08:54:57 +0000 Subject: [PATCH 69/73] add --- .../text_normalization/pl/data/numbers/hundreds_noun.tsv | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv new file mode 100644 index 000000000..f4e292b84 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/numbers/hundreds_noun.tsv @@ -0,0 +1,9 @@ +setka 1 +dwusetka 2 +trzysetka 3 +czterysetka 4 +pięćsetka 5 +sześćsetka 6 +siedemsetka 7 +osiemsetka 8 +dziewięćsetka 9 From 57783d234946b1a449ee67b2040d1ac472880308 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 5 Aug 2025 10:36:31 +0000 Subject: [PATCH 70/73] extend --- .../text_normalization/pl/graph_utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/graph_utils.py b/nemo_text_processing/text_normalization/pl/graph_utils.py index 4a296e5c2..1a7aa2387 100644 --- a/nemo_text_processing/text_normalization/pl/graph_utils.py +++ b/nemo_text_processing/text_normalization/pl/graph_utils.py @@ -69,10 +69,17 @@ def _load_roman(file: str): return graph @ fst -def all_to_graph(graph_dict, deterministic=False): - for default_key in ["mi_sg_nom", "sg_nom", "nom"]: - if default_key in graph_dict: - break +def all_to_graph(graph_dict, default=None, deterministic=False): + """ + Converts a dictionary of graphs to a single graph. + Polish has multiple cases, so this is useful for generating a single graph + """ + if default is None: + for default_key in ["mi_sg_nom", "sg_nom", "nom", "mi_pl_nom", "pl_nom"]: + if default_key in graph_dict: + break + else: + default_key = default output_graph = graph_dict[default_key] if not deterministic: for key in graph_dict: From dc3a3aa2e5f4d1622cde662a60fcd623bcd597c8 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Tue, 5 Aug 2025 10:36:48 +0000 Subject: [PATCH 71/73] more --- .../text_normalization/pl/taggers/cardinal.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index bc3cb7dc5..0cf1c31b1 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -228,18 +228,20 @@ def __init__(self, deterministic: bool = True): dwa_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "mi_pl_ins", "pl_gen", "mi_pl_nom"] pl_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] qnt_cases = ["mi_pl_nom", "pl_gen", "pl_gen", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] - jeden_filt = {} - jeden_compound = {} + # jeden (one) does not inflect in compound numbers, so we use the nominative form # e.g., https://www.poradnia-jezykowa.uni.lodz.pl/szczegoly/jeden-w-liczebnikach-wielowyrazowych # but a lot of people get this wrong, so we also include the inflected forms # This is different from Russian; also, jeden in compounds is a quantity, not singular + jeden_filt = {} + jeden_compound = {} for case in CASES: jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] jeden_compound[case] = jeden_all[f'mi_sg_nom'] if not deterministic: jeden_compound[case] |= self.jeden_all[f'mi_sg_{case}'] - # something similar for dwa + + # 2-4 are plural (5-9 are quantities) digit_forms_all = get_digit_forms("data/numbers/digit_forms.tsv") digit_graph = dict_to_graph(digit_forms_all, deterministic=deterministic) digit_pl = {} @@ -249,8 +251,8 @@ def __init__(self, deterministic: bool = True): digit_graph["3"][pl_cases[idx]], digit_graph["4"][pl_cases[idx]] ).optimize() - + digit_qnt = {} # zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) From fffcb6d6037fbde731528a90d8dce552772ede53 Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Fri, 7 Aug 2026 23:51:12 +0100 Subject: [PATCH 72/73] Add Polish text normalization Signed-off-by: Jim O'Regan --- .../text_normalization/normalize.py | 3 + .../text_normalization/pl/data/__init__.py | 2 +- .../pl/data/abbreviations.tsv | 1 + .../data/abbreviations_adjective_nondet.tsv | 3 + .../pl/data/abbreviations_nondet.tsv | 1 + .../pl/data/dates/__init__.py | 13 + .../pl/data/dates/month_abbr.tsv | 12 + .../pl/data/dates/months.tsv | 12 + .../pl/data/dates/months_roman.tsv | 12 + .../pl/data/grammar/__init__.py | 2 +- .../pl/data/grammar/noun_mi_c.tsv | 14 + .../pl/data/grammar/noun_mi_rok.tsv | 14 + .../pl/data/grammar/noun_mi_tr.tsv | 14 + .../pl/data/grammar/noun_mp_d.tsv | 14 + .../pl/data/grammar/noun_mp_el.tsv | 14 + .../pl/data/grammar/noun_mp_k.tsv | 14 + .../pl/data/grammar/noun_mp_p.tsv | 14 + .../pl/data/grammar/prepositions.tsv | 21 + .../pl/data/measures/__init__.py | 13 + .../pl/data/measures/units.tsv | 5 + .../pl/data/numbers/__init__.py | 2 +- .../pl/data/ordinal/__init__.py | 2 +- .../pl/data/roman/__init__.py | 13 + .../pl/data/roman/digit.tsv | 9 + .../pl/data/roman/hundreds.tsv | 9 + .../pl/data/roman/names.tsv | 12 + .../text_normalization/pl/data/roman/ties.tsv | 9 + .../text_normalization/pl/data/whitelist.tsv | 9 + .../text_normalization/pl/inflection.py | 116 +++ .../text_normalization/pl/licence | 13 + .../pl/taggers/abbreviation.py | 38 + .../text_normalization/pl/taggers/cardinal.py | 764 +++++++----------- .../text_normalization/pl/taggers/date.py | 92 +++ .../text_normalization/pl/taggers/measure.py | 72 ++ .../text_normalization/pl/taggers/ordinal.py | 139 ++-- .../text_normalization/pl/taggers/roman.py | 68 ++ .../text_normalization/pl/taggers/time.py | 71 ++ .../pl/taggers/tokenize_and_classify.py | 86 ++ .../tokenize_and_classify_with_audio.py | 153 ++++ .../pl/taggers/whitelist.py | 67 ++ .../text_normalization/pl/taggers/word.py | 24 + .../text_normalization/pl/utils.py | 13 +- .../pl/verbalizers/__init__.py | 13 + .../pl/verbalizers/cardinal.py | 25 + .../text_normalization/pl/verbalizers/date.py | 29 + .../pl/verbalizers/measure.py | 34 + .../pl/verbalizers/ordinal.py | 24 + .../pl/verbalizers/roman.py | 25 + .../text_normalization/pl/verbalizers/time.py | 43 + .../pl/verbalizers/verbalize.py | 35 + .../pl/verbalizers/verbalize_final.py | 51 ++ tests/nemo_text_processing/pl/__init__.py | 13 + .../pl/data_text_normalization/__init__.py | 13 + .../test_cases_cardinal.txt | 15 + .../test_cases_date.txt | 6 + .../test_cases_measure.txt | 6 + .../test_cases_normalize_with_audio.txt | 27 + .../test_cases_ordinal.txt | 9 + .../test_cases_roman.txt | 18 + .../test_cases_time.txt | 5 + .../test_cases_whitelist.txt | 12 + .../pl/test_abbreviation.py | 36 + .../pl/test_audio_normalizer.py | 37 + .../nemo_text_processing/pl/test_cardinal.py | 57 ++ tests/nemo_text_processing/pl/test_date.py | 60 ++ tests/nemo_text_processing/pl/test_measure.py | 31 + .../pl/test_normalization_with_audio.py | 33 + tests/nemo_text_processing/pl/test_ordinal.py | 39 + tests/nemo_text_processing/pl/test_roman.py | 42 + .../pl/test_sparrowhawk_normalization.sh | 63 ++ tests/nemo_text_processing/pl/test_time.py | 31 + .../nemo_text_processing/pl/test_whitelist.py | 61 ++ .../pynini_export.py | 6 + 73 files changed, 2331 insertions(+), 547 deletions(-) create mode 100644 nemo_text_processing/text_normalization/pl/data/abbreviations.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/dates/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/dates/months.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/measures/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/data/measures/units.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/roman/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/data/roman/digit.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/roman/names.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/roman/ties.tsv create mode 100644 nemo_text_processing/text_normalization/pl/data/whitelist.tsv create mode 100644 nemo_text_processing/text_normalization/pl/inflection.py create mode 100644 nemo_text_processing/text_normalization/pl/licence create mode 100644 nemo_text_processing/text_normalization/pl/taggers/abbreviation.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/roman.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/pl/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/roman.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py create mode 100644 tests/nemo_text_processing/pl/__init__.py create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/__init__.py create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/pl/test_abbreviation.py create mode 100644 tests/nemo_text_processing/pl/test_audio_normalizer.py create mode 100644 tests/nemo_text_processing/pl/test_cardinal.py create mode 100644 tests/nemo_text_processing/pl/test_date.py create mode 100644 tests/nemo_text_processing/pl/test_measure.py create mode 100644 tests/nemo_text_processing/pl/test_normalization_with_audio.py create mode 100644 tests/nemo_text_processing/pl/test_ordinal.py create mode 100644 tests/nemo_text_processing/pl/test_roman.py create mode 100644 tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/pl/test_time.py create mode 100644 tests/nemo_text_processing/pl/test_whitelist.py diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..6e803bf24 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -153,6 +153,9 @@ def __init__( elif lang == 'hu': from nemo_text_processing.text_normalization.hu.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.hu.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'pl': + from nemo_text_processing.text_normalization.pl.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.pl.verbalizers.verbalize_final import VerbalizeFinalFst elif lang == 'zh': from nemo_text_processing.text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.zh.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/pl/data/__init__.py b/nemo_text_processing/text_normalization/pl/data/__init__.py index 341a77c5b..6ebc808fa 100644 --- a/nemo_text_processing/text_normalization/pl/data/__init__.py +++ b/nemo_text_processing/text_normalization/pl/data/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/text_normalization/pl/data/abbreviations.tsv b/nemo_text_processing/text_normalization/pl/data/abbreviations.tsv new file mode 100644 index 000000000..df8e38c75 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/abbreviations.tsv @@ -0,0 +1 @@ +s-ka spółka noun_f_ka.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv b/nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv new file mode 100644 index 000000000..63c3c6f17 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/abbreviations_adjective_nondet.tsv @@ -0,0 +1,3 @@ +tzw. tak zwany +ur. urodzony +ang. angielski diff --git a/nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv b/nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv new file mode 100644 index 000000000..1c09cac2d --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/abbreviations_nondet.tsv @@ -0,0 +1 @@ +r. rok noun_mi_rok.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/dates/__init__.py b/nemo_text_processing/text_normalization/pl/data/dates/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv b/nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv new file mode 100644 index 000000000..66b0c5406 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/month_abbr.tsv @@ -0,0 +1,12 @@ +sty stycznia +lut lutego +mar marca +kwi kwietnia +maj maja +cze czerwca +lip lipca +sie sierpnia +wrz września +paź października +lis listopada +gru grudnia diff --git a/nemo_text_processing/text_normalization/pl/data/dates/months.tsv b/nemo_text_processing/text_normalization/pl/data/dates/months.tsv new file mode 100644 index 000000000..d8c99aee5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 stycznia +2 lutego +3 marca +4 kwietnia +5 maja +6 czerwca +7 lipca +8 sierpnia +9 września +10 października +11 listopada +12 grudnia diff --git a/nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv b/nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv new file mode 100644 index 000000000..152aa9dce --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/dates/months_roman.tsv @@ -0,0 +1,12 @@ +I stycznia +II lutego +III marca +IV kwietnia +V maja +VI czerwca +VII lipca +VIII sierpnia +IX września +X października +XI listopada +XII grudnia diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py b/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py index 341a77c5b..6ebc808fa 100644 --- a/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py +++ b/nemo_text_processing/text_normalization/pl/data/grammar/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv new file mode 100644 index 000000000..e2ef6ef01 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_c.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc +sg_ins em +sg_loc u +sg_voc u +pl_nom e +pl_gen ów +pl_dat om +pl_acc e +pl_ins ami +pl_loc ach +pl_voc e diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv new file mode 100644 index 000000000..990a46dd0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_rok.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen u +sg_dat owi +sg_acc +sg_ins iem +sg_loc u +sg_voc u +pl_nom i +pl_gen ów +pl_dat om +pl_acc i +pl_ins ami +pl_loc ach +pl_voc i diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv new file mode 100644 index 000000000..fba1a3200 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mi_tr.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc +sg_ins em +sg_loc ze +sg_voc ze +pl_nom y +pl_gen ów +pl_dat om +pl_acc y +pl_ins ami +pl_loc ach +pl_voc y diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv new file mode 100644 index 000000000..fcaa88061 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_d.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc zie +sg_voc zie +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv new file mode 100644 index 000000000..a9b2ca611 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_el.tsv @@ -0,0 +1,14 @@ +sg_nom eł +sg_gen ła +sg_dat łowi +sg_acc ła +sg_ins łem +sg_loc le +sg_voc le +pl_nom łowie +pl_gen łów +pl_dat łom +pl_acc łów +pl_ins łami +pl_loc łach +pl_voc łowie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv new file mode 100644 index 000000000..c35470aa7 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_k.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins iem +sg_loc u +sg_voc u +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv new file mode 100644 index 000000000..0f86fa546 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/noun_mp_p.tsv @@ -0,0 +1,14 @@ +sg_nom +sg_gen a +sg_dat owi +sg_acc a +sg_ins em +sg_loc ie +sg_voc ie +pl_nom owie +pl_gen ów +pl_dat om +pl_acc ów +pl_ins ami +pl_loc ach +pl_voc owie diff --git a/nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv b/nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv new file mode 100644 index 000000000..59865cf42 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/grammar/prepositions.tsv @@ -0,0 +1,21 @@ +bez gen +dla gen +do gen +od gen +u gen +z gen,ins +dzięki dat +ku dat +przeciw dat +wbrew dat +na acc,loc +o acc,loc +po acc,loc +przez acc +nad acc,ins +pod acc,ins +przed acc,ins +za acc,ins +między acc,ins +przy loc +w loc diff --git a/nemo_text_processing/text_normalization/pl/data/measures/__init__.py b/nemo_text_processing/text_normalization/pl/data/measures/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/measures/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/measures/units.tsv b/nemo_text_processing/text_normalization/pl/data/measures/units.tsv new file mode 100644 index 000000000..843f39f6f --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/measures/units.tsv @@ -0,0 +1,5 @@ +Hz herc noun_mi_c.tsv +kHz kiloherc noun_mi_c.tsv +m metr noun_mi_tr.tsv +cm centymetr noun_mi_tr.tsv +mm milimetr noun_mi_tr.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py index 341a77c5b..6ebc808fa 100644 --- a/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py +++ b/nemo_text_processing/text_normalization/pl/data/numbers/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py index 341a77c5b..6ebc808fa 100644 --- a/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py +++ b/nemo_text_processing/text_normalization/pl/data/ordinal/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/text_normalization/pl/data/roman/__init__.py b/nemo_text_processing/text_normalization/pl/data/roman/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/data/roman/digit.tsv b/nemo_text_processing/text_normalization/pl/data/roman/digit.tsv new file mode 100644 index 000000000..20a6ea627 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/digit.tsv @@ -0,0 +1,9 @@ +i 1 +ii 2 +iii 3 +iv 4 +v 5 +vi 6 +vii 7 +viii 8 +ix 9 diff --git a/nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv b/nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv new file mode 100644 index 000000000..af2100b79 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/hundreds.tsv @@ -0,0 +1,9 @@ +c 1 +cc 2 +ccc 3 +cd 4 +d 5 +dc 6 +dcc 7 +dccc 8 +cm 9 diff --git a/nemo_text_processing/text_normalization/pl/data/roman/names.tsv b/nemo_text_processing/text_normalization/pl/data/roman/names.tsv new file mode 100644 index 000000000..1bb422766 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/names.tsv @@ -0,0 +1,12 @@ +king Ludwik noun_mp_k.tsv +king Henryk noun_mp_k.tsv +king Karol noun_mp_rz.tsv +king Filip noun_mp_p.tsv +king Jan noun_mp_p.tsv +king Ferdynand noun_mp_d.tsv +king Napoleon noun_mp_p.tsv +king Jerzy noun_mp_y.tsv +king Franciszek Józef noun_mp_ek.tsv,noun_mp_p.tsv +queen Katarzyna noun_f_na.tsv +pope Klemens noun_mp_p.tsv +pope Jan Paweł noun_mp_p.tsv,noun_mp_el.tsv diff --git a/nemo_text_processing/text_normalization/pl/data/roman/ties.tsv b/nemo_text_processing/text_normalization/pl/data/roman/ties.tsv new file mode 100644 index 000000000..2b9e48e21 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/roman/ties.tsv @@ -0,0 +1,9 @@ +x 1 +xx 2 +xxx 3 +xl 4 +l 5 +lx 6 +lxx 7 +lxxx 8 +xc 9 diff --git a/nemo_text_processing/text_normalization/pl/data/whitelist.tsv b/nemo_text_processing/text_normalization/pl/data/whitelist.tsv new file mode 100644 index 000000000..feb31069b --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/data/whitelist.tsv @@ -0,0 +1,9 @@ +n.p.m. nad poziomem morza +tj. to jest +np. na przykład +itd. i tak dalej +m.in. między innymi +im. imienia +ur. urodzony +ang. angielski +tzw. tak zwany diff --git a/nemo_text_processing/text_normalization/pl/inflection.py b/nemo_text_processing/text_normalization/pl/inflection.py new file mode 100644 index 000000000..aa14a3715 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/inflection.py @@ -0,0 +1,116 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import pynini +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels +from pynini.lib import pynutil + + +def _load_endings(grammar_file: str) -> Dict[str, str]: + return { + slot: "" if ending == "" else ending + for slot, ending in load_labels(get_abs_path(f"data/grammar/{grammar_file}")) + } + + +def inflect_noun(word: str, grammar_file: str) -> Dict[str, str]: + """Inflects a noun using a grammar file containing slot-to-ending mappings.""" + + endings = _load_endings(grammar_file) + lemma_ending = endings["sg_nom"] + if not word.endswith(lemma_ending): + raise ValueError(f"{word!r} must end in {lemma_ending!r} from {grammar_file}") + stem = word[: -len(lemma_ending)] if lemma_ending else word + return {slot: stem + ending for slot, ending in endings.items()} + + +def case_prepositions() -> Dict[str, 'pynini.FstLike']: + """Loads prepositions as case-indexed identity graphs.""" + + graphs = {} + for preposition, cases in load_labels(get_abs_path("data/grammar/prepositions.tsv")): + graph = pynini.accep(preposition) + pynutil.delete(" ") + pynutil.insert(" ") + for case in cases.split(","): + graphs[case] = graph if case not in graphs else graphs[case] | graph + return {case: graph.optimize() for case, graph in graphs.items()} + + +def inflect_abbreviation(abbreviation: str, word: str, grammar_file: str) -> Dict[str, 'pynini.FstLike']: + """Creates abbreviation-to-word graphs for every slot in a nominal paradigm.""" + + endings = _load_endings(grammar_file) + lemma_ending = endings["sg_nom"] + if not abbreviation.endswith(lemma_ending) or not word.endswith(lemma_ending): + raise ValueError( + f"{abbreviation!r} and {word!r} must share the {lemma_ending!r} ending from {grammar_file}" + ) + abbreviation_stem = abbreviation[: -len(lemma_ending)] if lemma_ending else abbreviation + word_stem = word[: -len(lemma_ending)] if lemma_ending else word + return { + slot: pynini.cross(abbreviation_stem + ending, word_stem + ending).optimize() + for slot, ending in endings.items() + } + + +def expand_abbreviation(abbreviation: str, word: str, grammar_file: str) -> Dict[str, 'pynini.FstLike']: + """Creates graphs from one ambiguous abbreviation to each singular word form.""" + + endings = _load_endings(grammar_file) + lemma_ending = endings["sg_nom"] + if not word.endswith(lemma_ending): + raise ValueError(f"{word!r} must end in {lemma_ending!r} from {grammar_file}") + word_stem = word[: -len(lemma_ending)] if lemma_ending else word + return { + slot: pynini.cross(abbreviation, word_stem + ending).optimize() + for slot, ending in endings.items() + if slot.startswith("sg_") + } + + +def load_inflected_abbreviations(filepath: str) -> Dict[str, 'pynini.FstLike']: + """Loads abbreviation, lemma, and grammar triples into slot-indexed graphs.""" + + graphs = {} + for abbreviation, word, grammar_file in load_labels(get_abs_path(filepath)): + for slot, graph in inflect_abbreviation(abbreviation, word, grammar_file).items(): + graphs[slot] = graph if slot not in graphs else graphs[slot] | graph + return {slot: graph.optimize() for slot, graph in graphs.items()} + + +def load_ambiguous_abbreviations(filepath: str) -> Dict[str, 'pynini.FstLike']: + """Loads abbreviation, lemma, and grammar triples as singular alternatives.""" + + graphs = {} + for abbreviation, word, grammar_file in load_labels(get_abs_path(filepath)): + for slot, graph in expand_abbreviation(abbreviation, word, grammar_file).items(): + graphs[slot] = graph if slot not in graphs else graphs[slot] | graph + return {slot: graph.optimize() for slot, graph in graphs.items()} + + +def load_adjective_abbreviations(filepath: str) -> Dict[str, 'pynini.FstLike']: + """Loads abbreviations whose adjective component exposes every inflectional slot.""" + + from nemo_text_processing.text_normalization.pl.taggers.ordinal import complete_paradigm + + graphs = {} + for abbreviation, prefix, adjective in load_labels(get_abs_path(filepath)): + forms = adjective_inflection(adjective) + complete_paradigm(forms, complete=True) + for slot, form in forms.items(): + spoken = f"{prefix} {form}" if prefix else form + graph = pynini.cross(abbreviation, spoken) + graphs[slot] = graph if slot not in graphs else graphs[slot] | graph + return {slot: graph.optimize() for slot, graph in graphs.items()} diff --git a/nemo_text_processing/text_normalization/pl/licence b/nemo_text_processing/text_normalization/pl/licence new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/licence @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py b/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py new file mode 100644 index 000000000..382da8319 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.pl.graph_utils import PL_UPPER +from pynini.lib import pynutil + + +class AbbreviationFst(GraphFst): + """Classifies uppercase Polish initialisms, with or without dots.""" + + def __init__(self, whitelist=None, deterministic: bool = True): + super().__init__(name="abbreviation", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + graph = PL_UPPER + dot + pynini.closure(insert_space + PL_UPPER + dot, 1) + graph |= PL_UPPER + dot + pynini.closure(PL_UPPER + dot, 1) + graph |= PL_UPPER + pynini.closure(insert_space + PL_UPPER, 1) + + if whitelist is not None: + graph = pynini.compose( + pynini.difference(pynini.project(graph, "input"), pynini.project(whitelist.graph, "input")), graph + ) + + graph = pynutil.insert('value: "') + graph.optimize() + pynutil.insert('"') + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 0cf1c31b1..6d22d2fff 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (c) 2022, 2023 Jim O'Regan for Språkbanken Tal # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,490 +12,340 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Iterable + import pynini -from nemo_text_processing.text_normalization.en.graph_utils import ( - NEMO_DIGIT, - NEMO_SIGMA, - NEMO_SPACE, - NEMO_WHITE_SPACE, - GraphFst, - delete_space, - insert_space, -) +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, delete_space from nemo_text_processing.text_normalization.pl.graph_utils import PL_ALPHA from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels -from nemo_text_processing.text_normalization.pl.taggers.ordinal import complete_paradigm from pynini.lib import pynutil CASES = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] - - -def make_million(number: str, non_zero_pl: 'pynini.FstLike', non_zero_quant: 'pynini.FstLike', case: str = None, deterministic: bool = True) -> 'pynini.FstLike': - """ - Helper function for thousands/millions/milliards and higher - Args: - number: the string of the number - non_zero_pl: An fst of digits excluding 0, 1, 5-9, to prefix to plural forms (nom/acc) - non_zero_quant: An fst of digits excluding 0 and 1-4, to prefix to the quantity forms (nom/acc) - case: the string of the case (if None, nominative/accusative is presumed) - deterministic: if True, generate a deterministic fst - - Returns: - graph: A pynini.FstLike object - """ - if case is None: - sg_end = "" - pl_end = "y" - quant_end = "ów" - one = "jeden" - else: - SG = { - "loc": "ie", - "ins": "em", - "dat": "owi", - "gen": "a", - } - PL = { - "loc": "ach", - "ins": "ami", - "dat": "om", - "gen": "ów", - } - ONE = { - "loc": "jednym", - "ins": "jednym", - "dat": "jednemu", - "gen": "jednego", - } - sg_end = SG[case] - pl_end = PL[case] - one = ONE[case] - quant_end = pl_end - if case == "loc" and number.endswith("ard"): - sg_end = "zie" - graph = pynutil.add_weight(pynini.cross("001", f"{number}{sg_end}"), -0.001) - if not deterministic: - graph |= pynutil.add_weight(pynini.cross("001", f"{one} {number}{sg_end}"), -0.001) - graph |= non_zero_pl + pynutil.insert(f" {number}{pl_end}") - # hack for the stem change in tysiąc (1000) - if number == "tysiąc": - number = "tysięc" - graph |= non_zero_quant + pynutil.insert(f" {number}{quant_end}") - graph |= pynutil.delete("000") - graph += insert_space - return graph +DEFAULT_SLOT = "mi_sg_nom" +SCALE_NAMES = ["tysiąc", "milion", "miliard", "bilion", "biliard", "trylion", "tryliard"] def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': - """ - Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by space) - and converts to a string of digits: - "1 000" -> "1000" - Args: - fst: Any pynini.FstLike object. Function composes fst onto string parser fst - - Returns: - fst: A pynini.FstLike object - """ - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three - up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string - - cardinal_separator = NEMO_SPACE - cardinal_string = pynini.closure( - NEMO_DIGIT, 1 - ) # For string w/o punctuation (used for page numbers, thousand series) - + exactly_three_digits = NEMO_DIGIT**3 + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) + cardinal_string = pynini.closure(NEMO_DIGIT, 1) cardinal_string |= ( up_to_three_digits - + pynutil.delete(cardinal_separator) - + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + pynutil.delete(" ") + + pynini.closure(exactly_three_digits + pynutil.delete(" ")) + exactly_three_digits ) - return cardinal_string @ fst -def make_inflected_graph_dict(file_path: str, cross: str, deterministic=False) -> dict: - graph_dict = {} - for line in load_labels(get_abs_path(file_path)): - key, value = line[0], line[1] - if key not in graph_dict: - graph_dict[key] = pynini.cross(cross, value) - else: - if not deterministic: - graph_dict[key] |= pynini.cross(cross, value) - return graph_dict - - -def get_nominal_inflections(inflection_file, noun_file): +def get_digit_forms(filepath: str) -> Dict[str, Dict[str, object]]: output = {} - inflections = {a[0]: a[1] for a in load_labels(get_abs_path(inflection_file))} - digit_noun_tsv = load_labels(get_abs_path(noun_file)) - for digit_noun in digit_noun_tsv: - word = digit_noun[0] - digit = digit_noun[1] - lemma_ending = inflections["sg_nom"] - assert word.endswith(lemma_ending), f"Word {word} does not end with {lemma_ending}" - stem = word[:-len(lemma_ending)] - wordforms = {k: stem + v for k, v in inflections.items()} - output[digit] = wordforms - return output - - -def get_nominal_graph(inflection_file, noun_file): - output = {} - input = get_nominal_inflections(inflection_file, noun_file) - for item in input: - for key in input[item]: - if not key in output: - output[key] = pynini.cross(item, input[item][key]) - else: - output[key] |= pynini.cross(item, input[item][key]) - return output - - -def get_digit_forms(filepath): - """ - Returns a dictionary of digit forms for Polish numbers. - """ - output = {} - for line in load_labels(get_abs_path(filepath)): - digit, grammar, form = line[0], line[1], line[2] - if not digit in output: - output[digit] = {} - if grammar not in output[digit]: - output[digit][grammar] = form + for digit, grammar, form in load_labels(get_abs_path(filepath)): + forms = output.setdefault(digit, {}) + if grammar not in forms: + forms[grammar] = form + elif isinstance(forms[grammar], list): + forms[grammar].append(form) else: - if type(output[digit][grammar]) is list: - output[digit][grammar].append(form) - else: - output[digit][grammar] = [output[digit][grammar], form] + forms[grammar] = [forms[grammar], form] return output -def dict_to_graph(input_dict: dict, deterministic: bool = True) -> dict: - """ - Converts a nested dictionary of forms to a dict of pynini.FSTs. - Example input: - {'2': {'mi_pl_ins': ['form1', 'form2'], 'mi_sg_nom': 'form3'}} - Output: - {'2': {'mi_pl_ins': FST, 'mi_sg_nom': FST}} - """ - graph_dict = {} - for key, value in input_dict.items(): - graph_dict[key] = {} - for subkey, subvalue in value.items(): - if isinstance(subvalue, list): - graph = pynini.cross(key, subvalue[0]) - if not deterministic: - for alt in subvalue[1:]: - graph |= pynini.cross(key, alt) - else: - graph = pynini.cross(key, subvalue) - graph_dict[key][subkey] = graph - return graph_dict +def _forms_to_graphs( + forms: Dict[str, Dict[str, object]], deterministic: bool +) -> Dict[str, Dict[str, 'pynini.FstLike']]: + graphs = {} + for number, slots in forms.items(): + graphs[number] = {} + for slot, values in slots.items(): + values = values if isinstance(values, list) else [values] + if deterministic: + values = values[:1] + graphs[number][slot] = pynini.union(*(pynini.cross(number, value) for value in values)).optimize() + return graphs + + +def _invert_string_file(path: str) -> 'pynini.FstLike': + return pynini.invert(pynini.string_file(get_abs_path(path))).optimize() + + +def _case_for_slot(slot: str) -> str: + if slot == "compound": + return slot + for case in CASES: + if slot == case or slot.endswith(f"_{case}"): + return case + raise ValueError(f"Cannot determine case from slot: {slot}") + + +def _select(mapping: Dict[str, 'pynini.FstLike'], keys: Iterable[str]) -> 'pynini.FstLike': + for key in keys: + if key in mapping: + return mapping[key] + raise KeyError(f"None of {list(keys)} is available") + + +def _noun_forms(lemma: str) -> Dict[str, str]: + if lemma == "tysiąc": + return {key: value for key, value in load_labels(get_abs_path("data/numbers/tysiac.tsv"))} + + stem = lemma + loc_sg = "u" + if lemma.endswith("ion"): + loc_sg = "ie" + elif lemma.endswith("iard"): + loc_sg = "zie" + return { + "sg_nom": lemma, + "sg_gen": lemma + "a", + "sg_dat": lemma + "owi", + "sg_acc": lemma, + "sg_ins": lemma + "em", + "sg_loc": stem + loc_sg, + "sg_voc": lemma + "ie", + "pl_nom": lemma + "y", + "pl_gen": lemma + "ów", + "pl_dat": lemma + "om", + "pl_acc": lemma + "y", + "pl_ins": lemma + "ami", + "pl_loc": lemma + "ach", + "pl_voc": lemma + "y", + } class CardinalFst(GraphFst): - """ - Finite state transducer for classifying cardinals, e.g. - "1000" -> cardinal { integer: "tysiąc" } - "2 000 000" -> cardinal { integer: "dwa miliony" } - - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + """Classifies Polish cardinal numbers and exposes each inflectional graph in ``graphs``.""" def __init__(self, deterministic: bool = True): super().__init__(name="cardinal", kind="classify", deterministic=deterministic) - jeden_all = adjective_inflection("jeden") - jeden_graph = pynini.cross("1", jeden_all["mi_sg_nom"]) - # in compound numbers, jeden does not inflect - jeden_only = pynini.cross("1", jeden_all["mi_sg_nom"]) + digit_forms = get_digit_forms("data/numbers/digit_forms.tsv") + teen_forms = get_digit_forms("data/numbers/teens_forms.tsv") + digit_graphs = _forms_to_graphs(digit_forms, deterministic) + teen_graphs = _forms_to_graphs(teen_forms, deterministic) + + jeden = adjective_inflection("jeden", compound="jedno") + from nemo_text_processing.text_normalization.pl.taggers.ordinal import complete_paradigm + + complete_paradigm(jeden, complete=True) + self.jeden_all = {slot: pynini.cross("1", form) for slot, form in jeden.items()} + + zero_forms = { + "sg_nom": "zero", + "sg_gen": "zera", + "sg_dat": "zeru", + "sg_acc": "zero", + "sg_ins": "zerem", + "sg_loc": "zerze", + "sg_voc": "zero", + } + self.zero_all = {slot: pynini.cross("0", form) for slot, form in zero_forms.items()} + self.zero_sg = {slot[3:]: graph for slot, graph in self.zero_all.items()} + + ordinary_slots = set(self.jeden_all) + for forms in digit_forms.values(): + ordinary_slots.update(forms) + for forms in teen_forms.values(): + ordinary_slots.update(forms) + + tens_nom = _invert_string_file("data/numbers/tens.tsv") + tens_gen = _invert_string_file("data/numbers/tens_gen.tsv") + tens_ins = _invert_string_file("data/numbers/tens_ins.tsv") + tens_compound = _invert_string_file("data/numbers/tens_prefix.tsv") + hundreds_nom = _invert_string_file("data/numbers/hundreds.tsv") + hundreds_gen = _invert_string_file("data/numbers/hundreds_gen.tsv") + hundreds_ins = _invert_string_file("data/numbers/hundreds_ins.tsv") + hundreds_compound = _invert_string_file("data/numbers/hundreds.tsv") + + join = pynutil.insert(" ") + compound_join = pynutil.insert("") if not deterministic: - for key in jeden_all: - if key == "mi_sg_nom": - continue - jeden_graph |= pynini.cross("1", jeden_all[key]) - complete_paradigm(jeden_all) - self.jeden_all = {a[0]: pynini.cross("1", a[1]) for a in jeden_all.items()} - self.zero_all = get_nominal_graph("data/grammar/noun_nt_ro.tsv", "data/numbers/zero.tsv") - self.zero_sg = {x.replace("sg_", ""): y for x, y in self.zero_all.items() if x.startswith("sg_")} - - dwa_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "mi_pl_ins", "pl_gen", "mi_pl_nom"] - pl_cases = ["mi_pl_nom", "pl_gen", "pl_dat", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] - qnt_cases = ["mi_pl_nom", "pl_gen", "pl_gen", "mi_pl_nom", "pl_ins", "pl_gen", "mi_pl_nom"] - - # jeden (one) does not inflect in compound numbers, so we use the nominative form - # e.g., https://www.poradnia-jezykowa.uni.lodz.pl/szczegoly/jeden-w-liczebnikach-wielowyrazowych - # but a lot of people get this wrong, so we also include the inflected forms - # This is different from Russian; also, jeden in compounds is a quantity, not singular - jeden_filt = {} - jeden_compound = {} - for case in CASES: - jeden_filt[case] = self.jeden_all[f'mi_sg_{case}'] - jeden_compound[case] = jeden_all[f'mi_sg_nom'] - if not deterministic: - jeden_compound[case] |= self.jeden_all[f'mi_sg_{case}'] - - # 2-4 are plural (5-9 are quantities) - digit_forms_all = get_digit_forms("data/numbers/digit_forms.tsv") - digit_graph = dict_to_graph(digit_forms_all, deterministic=deterministic) - digit_pl = {} - for idx in range(len(CASES)): - digit_pl[CASES[idx]] = pynini.union( - digit_graph["2"][dwa_cases[idx]], - digit_graph["3"][pl_cases[idx]], - digit_graph["4"][pl_cases[idx]] + compound_join |= pynutil.add_weight(pynutil.insert(" "), 0.001) + + self.graphs = {} + self.two_digit_graphs = {} + self.hundreds_graphs = {} + + for slot in sorted(ordinary_slots): + case = _case_for_slot(slot) + if case == "compound": + tens = tens_compound + hundreds = hundreds_compound + component_join = compound_join + elif case == "ins": + tens = tens_ins + hundreds = hundreds_ins + component_join = join + elif case in {"gen", "dat", "loc"} or slot.startswith("mp_"): + tens = tens_gen + hundreds = hundreds_gen + component_join = join + else: + tens = tens_nom + hundreds = hundreds_nom + component_join = join + + digit = self._digit_for_slot(digit_graphs, slot) + compound_digit = self._compound_digit_for_slot(digit_graphs, slot) + teen = self._teen_for_slot(teen_graphs, slot) + isolated_one = self._one_for_slot(slot, compound=False, deterministic=deterministic) + compound_one = self._one_for_slot(slot, compound=True, deterministic=deterministic) + two_digit = ( + teen + | tens + pynutil.delete("0") + | tens + component_join + (compound_digit | compound_one) + | pynutil.delete("0") + (digit | isolated_one) + ).optimize() + hundred = ( + hundreds + pynutil.delete("00") + | hundreds + component_join + two_digit + | pynutil.delete("0") + two_digit + | pynutil.delete("00") + (digit | isolated_one) ).optimize() - - digit_qnt = {} - - - # zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) - # digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) - # teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) - # ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) - # hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) - - # plural_3digits = NEMO_DIGIT + (NEMO_DIGIT - "1") + pynini.union("2", "3", "4") - # quantity_3digits = NEMO_DIGIT + pynini.union( - # "1" + NEMO_DIGIT, - # (NEMO_DIGIT - "1") + pynini.union("0", "5", "6", "7", "8", "9") - # ) - - # # Any single digit - # graph_digit = digit - # digits_no_one = (NEMO_DIGIT - "1") @ graph_digit - # self.digit = graph_digit - - # single_digits_graph = graph_digit | zero - # self.single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) - - # # Any double digit - # graph_tens = teen - # graph_ties = ties - # if deterministic: - # graph_tens |= graph_ties + (pynutil.delete('0') | graph_digit) - # else: - # graph_tens |= pynutil.add_weight(pynini.cross("18", "aderton"), -0.001) - # graph_tens |= pynutil.add_weight( - # graph_ties + (pynutil.delete('0') | (graph_digit | insert_space + graph_digit)), -0.001 - # ) - - # hundreds = digits_no_one + pynutil.insert("hundra") - # hundreds |= pynini.cross("1", "hundra") - # if not deterministic: - # hundreds |= pynutil.add_weight(pynini.cross("1", "etthundra"), -0.001) - # hundreds |= pynutil.add_weight(digit + pynutil.insert(NEMO_SPACE) + pynutil.insert("hundra"), -0.001) - - # self.tens = graph_tens.optimize() - - # graph_two_digit_non_zero = pynini.union(graph_digit, graph_tens, (pynutil.delete("0") + graph_digit)) - # if not deterministic: - # graph_two_digit_non_zero |= pynutil.add_weight( - # pynini.union(graph_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + graph_digit)), -0.001 - # ) - - # self.two_digit_non_zero = graph_two_digit_non_zero.optimize() - - # graph_final_two_digit_non_zero = pynini.union(final_digit, graph_tens, (pynutil.delete("0") + final_digit)) - # if not deterministic: - # graph_final_two_digit_non_zero |= pynutil.add_weight( - # pynini.union(final_digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + final_digit)), -0.001 - # ) - - # self.final_two_digit_non_zero = graph_final_two_digit_non_zero.optimize() - - # # Three digit strings - # graph_hundreds = hundreds + pynini.union(pynutil.delete("00"), graph_tens, (pynutil.delete("0") + final_digit)) - # if not deterministic: - # graph_hundreds |= pynutil.add_weight( - # hundreds - # + pynini.union( - # pynutil.delete("00"), - # (graph_tens | pynutil.insert(NEMO_SPACE) + graph_tens), - # (pynini.cross("0", NEMO_SPACE) + final_digit), - # ), - # -0.001, - # ) - - # self.hundreds = graph_hundreds.optimize() - - # # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) - # graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) - - # graph_hundreds_component_at_least_one_non_zero_digit = graph_hundreds_component | ( - # pynutil.delete("00") + graph_digit - # ) - - # graph_hundreds_component_at_least_one_non_zero_digit_no_one = graph_hundreds_component | ( - # pynutil.delete("00") + digits_no_one - # ) - # self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( - # graph_hundreds_component_at_least_one_non_zero_digit_no_one.optimize() - # ) - - # tusen = pynutil.insert("tusen") - # etttusen = tusen - - # following_hundred = insert_space + graph_hundreds_component_at_least_one_non_zero_digit - # if not deterministic: - # following_hundred |= graph_hundreds_component_at_least_one_non_zero_digit - - # graph_thousands_component_at_least_one_non_zero_digit = pynini.union( - # pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit, - # graph_hundreds_component_at_least_one_non_zero_digit_no_one - # + tusen - # + (following_hundred | pynutil.delete("000")), - # pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), - # ) - # self.graph_thousands_component_at_least_one_non_zero_digit = ( - # graph_thousands_component_at_least_one_non_zero_digit.optimize() - # ) - - # graph_thousands_component_at_least_one_non_zero_digit_no_one = pynini.union( - # pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit_no_one, - # graph_hundreds_component_at_least_one_non_zero_digit_no_one - # + tusen - # + (following_hundred | pynutil.delete("000")), - # pynini.cross("001", etttusen) + (following_hundred | pynutil.delete("000")), - # ) - # self.graph_thousands_component_at_least_one_non_zero_digit_no_one = ( - # graph_thousands_component_at_least_one_non_zero_digit_no_one.optimize() - # ) - - # non_zero_no_one = graph_hundreds_component_at_least_one_non_zero_digit_no_one - # graph_million = make_million("milion", non_zero_no_one, deterministic) - # graph_milliard = make_million("miliard", non_zero_no_one, deterministic) - # graph_billion = make_million("bilion", non_zero_no_one, deterministic) - # graph_billiard = make_million("biliard", non_zero_no_one, deterministic) - # graph_trillion = make_million("trilion", non_zero_no_one, deterministic) - # graph_trilliard = make_million("triliard", non_zero_no_one, deterministic) - - # graph = ( - # graph_trilliard - # + graph_trillion - # + graph_billiard - # + graph_billion - # + graph_milliard - # + graph_million - # + (graph_thousands_component_at_least_one_non_zero_digit | pynutil.delete("000000")) - # ) - - # self.graph = ( - # ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) - # @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - # @ NEMO_DIGIT ** 24 - # @ graph - # @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) - # @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) - # @ pynini.cdrewrite( - # pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), PL_ALPHA, PL_ALPHA, NEMO_SIGMA - # ) - # ) - - # self.graph_hundreds_component_at_least_one_non_zero_digit = ( - # pynini.closure(NEMO_DIGIT, 2, 3) | pynini.difference(NEMO_DIGIT, pynini.accep("0")) - # ) @ self.graph - # self.graph_hundreds_component_at_least_one_non_zero_digit_en = ( - # self.graph_hundreds_component_at_least_one_non_zero_digit - # @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) - # ) - # # For plurals, because the 'one' in 21, etc. still needs to agree - # self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( - # pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit, "input") - "1" - # ) @ self.graph_hundreds_component_at_least_one_non_zero_digit - # self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en = ( - # pynini.project(self.graph_hundreds_component_at_least_one_non_zero_digit_en, "input") - "1" - # ) @ self.graph_hundreds_component_at_least_one_non_zero_digit_en - - # zero_space = zero + insert_space - # self.zero_space = zero_space - # self.three_digits_read = pynini.union( - # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - # zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), - # zero_space + zero_space + digit, - # ) - # self.three_digits_read_en = pynini.union( - # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - # zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), - # zero_space + zero_space + digit, - # ) - # self.three_digits_read_frac = pynini.union( - # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - # zero_space + digit + insert_space + digit, - # ) - # self.three_digits_read_frac_en = pynini.union( - # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - # zero_space + digit + insert_space + digit, - # ) - # self.two_or_three_digits_read_frac = pynini.union( - # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, - # zero_space + single_digits_graph + pynini.closure(insert_space + digit, 0, 1), - # single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), - # zero_space + zero_space + zero, - # single_digits_graph, - # ) - # self.two_or_three_digits_read_frac_en = pynini.union( - # ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - # @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), - # zero_space + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 0, 1), - # single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), - # zero_space + zero_space + zero, - # single_digits_graph, - # ) - # self.two_digits_read = pynini.union(((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, zero_space + digit) - # self.two_digits_read_en = pynini.union( - # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), - # zero_space + digit, - # ) - # self.any_read_digit = ((NEMO_DIGIT - "0") @ digit) + pynini.closure(insert_space + digit) - # if not deterministic: - # self.three_digits_read |= pynutil.add_weight(digit + insert_space + digit + insert_space + digit, -0.001) - # self.three_digits_read |= pynutil.add_weight( - # ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens + insert_space + digit, -0.001 - # ) - # self.three_digits_read |= pynutil.add_weight( - # digit + insert_space + ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, -0.001 - # ) - # self.two_digits_read |= pynutil.add_weight(digit + insert_space + digit, -0.001) - - # self.graph |= zero - - # self.graph_unfiltered = self.graph - # self.graph = filter_punctuation(self.graph).optimize() - # self.graph_en = self.graph @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA) - # self.graph_no_one = (pynini.project(self.graph, "input") - "1") @ self.graph - # self.graph_no_one_en = (pynini.project(self.graph_en, "input") - "1") @ self.graph_en - - # joiner_chars = pynini.union("-", "–", "—") - # joiner = pynini.cross(joiner_chars, " till ") - # self.range = self.graph + joiner + self.graph - # if not deterministic: - # either_one = self.graph | self.graph_en - # self.range = either_one + joiner + either_one - - # optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) - - # final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") - # if not deterministic: - # final_graph |= pynutil.add_weight( - # optional_minus_graph + pynutil.insert("integer: \"") + self.graph_en + pynutil.insert("\""), -0.001 - # ) - # final_graph |= pynutil.add_weight( - # pynutil.insert("integer: \"") + self.single_digits_graph + pynutil.insert("\""), -0.001 - # ) - - # final_graph = self.add_tokens(final_graph) - # self.fst = final_graph.optimize() + + self.two_digit_graphs[slot] = two_digit + self.hundreds_graphs[slot] = hundred + for slot, hundred in self.hundreds_graphs.items(): + self.graphs[slot] = self._make_full_number_graph(hundred, slot, deterministic) + + self.graph_dict = self.graphs + compound_boundary = pynutil.delete("-") + if not deterministic: + compound_boundary += pynini.union( + pynutil.insert(""), pynutil.add_weight(pynutil.insert(" "), 0.001) + ) + self.compound = ( + self.graphs["compound"] + compound_boundary + pynini.closure(PL_ALPHA, 1) + ).optimize() + + self.graph = filter_punctuation(self.graphs[DEFAULT_SLOT] | self.zero_all["sg_nom"]).optimize() | self.compound + if not deterministic: + self.graph = ( + filter_punctuation(pynini.union(*self.graphs.values(), *self.zero_all.values())).optimize() + | self.compound + ) + + self.graph_unfiltered = self.graph + optional_minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + final_graph = optional_minus + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + self.fst = self.add_tokens(final_graph).optimize() + + def _digit_for_slot(self, graphs, slot): + choices = { + "f_pl_nom": ["f_pl_nom", "mi_pl_nom"], + "f_pl_ins": ["f_pl_ins", "pl_ins"], + "mi_pl_ins": ["mi_pl_ins", "pl_ins"], + "pl_ins": ["mi_pl_ins", "pl_ins"], + }.get(slot, [slot]) + case = _case_for_slot(slot) + if case in {"acc", "voc"}: + choices += [slot.rsplit("_", 1)[0] + "_nom", "mi_pl_nom"] + choices += [f"pl_{case}", "mi_pl_nom"] + return pynini.union(*(_select(forms, choices) for forms in graphs.values())).optimize() + + def _compound_digit_for_slot(self, graphs, slot): + if slot.startswith("mp_") and _case_for_slot(slot) in {"nom", "acc"}: + choices = ["pl_gen", "mp_pl_nom"] + elif slot == "pl_ins": + choices = ["mi_pl_ins", "pl_ins"] + else: + choices = [slot] + case = _case_for_slot(slot) + choices += [f"pl_{case}", "mi_pl_nom"] + return pynini.union(*(_select(forms, choices) for forms in graphs.values())).optimize() + + def _teen_for_slot(self, graphs, slot): + case = _case_for_slot(slot) + choices = [slot, f"pl_{case}"] + if case in {"acc", "voc"}: + choices += ["mp_pl_nom" if slot.startswith("mp_") else "mi_pl_nom"] + choices += ["mi_pl_nom"] + return pynini.union(*(_select(forms, choices) for forms in graphs.values())).optimize() + + def _one_for_slot(self, slot, compound, deterministic): + if slot == "compound": + return self.jeden_all[slot] + if not compound: + return self.jeden_all[slot] + case = _case_for_slot(slot) + graph = pynini.cross("1", "jeden") + if not deterministic: + key = slot if slot in self.jeden_all else f"mi_sg_{case}" + graph |= pynutil.add_weight(self.jeden_all[key], 0.001) + return graph.optimize() + + def _make_full_number_graph(self, group, slot, deterministic): + case = _case_for_slot(slot) + if case == "compound": + short_input = pynini.closure(NEMO_DIGIT, 1, 3) + pad = short_input @ pynini.cdrewrite( + pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA + ) @ NEMO_DIGIT**3 + return (pad @ group).optimize() + + scale_slot = { + "nom": "mi_sg_nom", + "acc": "mi_sg_acc", + "voc": "mi_sg_voc", + "gen": "pl_gen", + "dat": "pl_dat", + "ins": "mi_pl_ins", + "loc": "pl_gen", + }[case] + scale_group = self.hundreds_graphs[scale_slot] + + plural_group = self._restrict_group(scale_group, "plural") + quantity_group = self._restrict_group(scale_group, "quantity") + non_one_group = self._restrict_group(scale_group, "non_one") + factors = [] + for scale in reversed(SCALE_NAMES): + forms = _noun_forms(scale) + if case in {"nom", "acc", "voc"}: + singular = forms[f"sg_{case}"] + plural = forms[f"pl_{case}"] + quantity = forms["pl_gen"] + factor = ( + pynutil.delete("000") + | pynini.cross("001", singular) + pynutil.insert(" ") + | plural_group + pynutil.insert(" " + plural + " ") + | quantity_group + pynutil.insert(" " + quantity + " ") + ) + else: + factor = ( + pynutil.delete("000") + | pynini.cross("001", forms[f"sg_{case}"]) + pynutil.insert(" ") + | non_one_group + pynutil.insert(" " + forms[f"pl_{case}"] + " ") + ) + if not deterministic: + factor |= pynutil.add_weight( + pynini.cross("001", "jeden " + forms[f"sg_{case}"] + " "), 0.001 + ) + factors.append(factor) + + padded = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**24 + ) + full = None + for factor in factors: + full = factor if full is None else full + factor + full += group | pynutil.delete("000") + clean = pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + return (padded @ full @ clean).optimize() + + @staticmethod + def _restrict_group(group, kind): + hundred = NEMO_DIGIT + if kind == "plural": + inputs = hundred + (NEMO_DIGIT - "1") + pynini.union("2", "3", "4") + elif kind == "quantity": + inputs = hundred + pynini.union( + "1" + NEMO_DIGIT, + (NEMO_DIGIT - "1") + pynini.union("0", "5", "6", "7", "8", "9"), + (NEMO_DIGIT - "0") + "1", + ) + else: + inputs = NEMO_DIGIT**3 - "001" - "000" + return inputs @ group diff --git a/nemo_text_processing/text_normalization/pl/taggers/date.py b/nemo_text_processing/text_normalization/pl/taggers/date.py new file mode 100644 index 000000000..33ad30e78 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/date.py @@ -0,0 +1,92 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.pl.utils import get_abs_path +from pynini.lib import pynutil + + +class DateFst(GraphFst): + """Classifies Polish dates and exposes case-coordinated graphs.""" + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + day_input = pynini.union(*(str(day) for day in range(1, 32))) + numeric_day_input = day_input | pynini.union(*(f"{day:02d}" for day in range(1, 10))) + + month_number = pynini.string_file(get_abs_path("data/dates/months.tsv")) + month_number = pynutil.delete("0") + month_number | month_number + month_words = pynini.project(month_number, "output") + month_abbr = pynini.string_file(get_abs_path("data/dates/month_abbr.tsv")) + month_roman = pynini.string_file(get_abs_path("data/dates/months_roman.tsv")) + + year_prefix = ( + (NEMO_DIGIT - "0") + pynutil.insert("000") + ) @ cardinal.graphs["mi_sg_nom"] + + month_numeric_field = pynutil.insert(' month: "') + month_number + pynutil.insert('"') + month_roman_field = pynutil.insert(' month: "') + month_roman + pynutil.insert('"') + month_word_field = ( + pynutil.insert(' month: "') + (month_words | month_abbr) + pynutil.insert('"') + ) + + self.graphs = {} + self.year_graphs = {} + for slot, ordinal_graph in ordinal.graphs.items(): + if slot == "compound": + continue + day = day_input @ ordinal_graph + numeric_day = numeric_day_input @ ordinal_graph + year = year_prefix + insert_space + (NEMO_DIGIT**3 @ ordinal_graph) + year_with_abbreviation = year + pynini.closure( + pynini.closure(delete_space, 0, 1) + pynini.cross("r.", " roku"), 0, 1 + ) + day_field = pynutil.insert('day: "') + day + pynutil.insert('"') + numeric_day_field = pynutil.insert('day: "') + numeric_day + pynutil.insert('"') + year_field = pynutil.insert(' year: "') + year_with_abbreviation + pynutil.insert('"') + + numeric = pynini.union( + *( + numeric_day_field + + pynutil.delete(separator) + + month_numeric_field + + pynutil.delete(separator) + + year_field + for separator in (".", "-", "/") + ) + ) + numeric |= ( + numeric_day_field + + pynutil.delete(".") + + month_roman_field + + pynutil.delete(".") + + year_field + ) + written = day_field + delete_space + month_word_field + written += pynini.closure(delete_space + year_field, 0, 1) + self.graphs[slot] = (numeric | written).optimize() + self.year_graphs[slot] = ( + pynutil.insert('year: "') + year + pynutil.insert('"') + ).optimize() + + self.graph_dict = self.graphs + if deterministic: + self.final_graph = self.graphs["mi_sg_gen"] + else: + self.final_graph = pynini.union( + *self.graphs.values(), *self.year_graphs.values() + ).optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/measure.py b/nemo_text_processing/text_normalization/pl/taggers/measure.py new file mode 100644 index 000000000..e8746bab4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/measure.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.text_normalization.pl.inflection import inflect_noun +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +def _case(slot: str) -> str: + for case in ("nom", "gen", "dat", "acc", "ins", "loc", "voc"): + if slot == case or slot.endswith(f"_{case}"): + return case + raise ValueError(f"Cannot determine case from {slot!r}") + + +class MeasureFst(GraphFst): + """Classifies integer measures with case-inflected masculine units.""" + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + unit_graphs = {} + for symbol, lemma, grammar_file in load_labels(get_abs_path("data/measures/units.tsv")): + for slot, form in inflect_noun(lemma, grammar_file).items(): + graph = pynini.cross(symbol, form) + unit_graphs[slot] = graph if slot not in unit_graphs else unit_graphs[slot] | graph + unit_graphs = {slot: graph.optimize() for slot, graph in unit_graphs.items()} + + positive = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) + one = pynini.accep("1") + few = pynini.intersect(positive, pynini.closure(NEMO_DIGIT) + pynini.union("2", "3", "4")) + few = pynini.difference(few, pynini.closure(NEMO_DIGIT) + pynini.union("12", "13", "14")) + many = pynini.union("0", pynini.difference(pynini.difference(positive, one), few)).optimize() + optional_space = pynini.closure(delete_space, 0, 1) + + def graph_for(number_input, number_graph, unit_slot): + return ( + pynutil.insert('cardinal { integer: "') + + (number_input @ number_graph) + + pynutil.insert('" } units: "') + + optional_space + + unit_graphs[unit_slot] + + pynutil.insert('"') + ) + + self.graphs = {} + for slot, number_graph in cardinal.graphs.items(): + if slot == "compound": + continue + case = _case(slot) + graph = graph_for(one, number_graph, f"sg_{case}") + graph |= graph_for(few, number_graph, f"pl_{case}") + graph |= graph_for(many, number_graph, "pl_gen") + self.graphs[slot] = graph.optimize() + + self.graph_dict = self.graphs + graph = self.graphs["mi_sg_nom"] if deterministic else pynini.union(*self.graphs.values()) + self.final_graph = graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index ddaa16a03..0b45d82ab 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,22 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space from nemo_text_processing.text_normalization.pl.graph_utils import all_to_graph -from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels -# from nemo_text_processing.text_normalization.pl.taggers.cardinal import cardinal_graph +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels from pynini.lib import pynutil -from nemo_text_processing.text_normalization.pl.utils import adjective_inflection - -def complete_paradigm(partial, complete=False): +def complete_paradigm(partial: Dict[str, str], complete: bool = False): partial["mi_sg_acc"] = partial["mi_sg_nom"] partial["mi_sg_loc"] = partial["mi_sg_ins"] partial["mi_sg_voc"] = partial["mi_sg_nom"] - # ma.sg same as mi.sg, except acc = gen partial["ma_sg_nom"] = partial["mi_sg_nom"] partial["ma_sg_gen"] = partial["mi_sg_gen"] partial["ma_sg_dat"] = partial["mi_sg_dat"] @@ -34,27 +31,18 @@ def complete_paradigm(partial, complete=False): partial["ma_sg_ins"] = partial["mi_sg_ins"] partial["ma_sg_loc"] = partial["mi_sg_loc"] partial["ma_sg_voc"] = partial["mi_sg_voc"] - # mp.sg same as ma.sg - partial["mp_sg_nom"] = partial["ma_sg_nom"] - partial["mp_sg_gen"] = partial["ma_sg_gen"] - partial["mp_sg_dat"] = partial["ma_sg_dat"] - partial["mp_sg_acc"] = partial["ma_sg_acc"] - partial["mp_sg_ins"] = partial["ma_sg_ins"] - partial["mp_sg_loc"] = partial["ma_sg_loc"] - partial["mp_sg_voc"] = partial["ma_sg_voc"] - # nt.sg same as mi.sg aside from nom/acc/voc + for case in ["nom", "gen", "dat", "acc", "ins", "loc", "voc"]: + partial[f"mp_sg_{case}"] = partial[f"ma_sg_{case}"] partial["nt_sg_gen"] = partial["mi_sg_gen"] partial["nt_sg_dat"] = partial["mi_sg_dat"] partial["nt_sg_acc"] = partial["nt_sg_nom"] partial["nt_sg_ins"] = partial["mi_sg_ins"] partial["nt_sg_loc"] = partial["mi_sg_loc"] partial["nt_sg_voc"] = partial["nt_sg_nom"] - # f.sg partial["f_sg_dat"] = partial["f_sg_gen"] partial["f_sg_acc"] = partial["f_sg_ins"] partial["f_sg_loc"] = partial["f_sg_gen"] partial["f_sg_voc"] = partial["f_sg_nom"] - # plurals partial["mp_pl_acc"] = partial["pl_loc"] partial["mp_pl_voc"] = partial["mp_pl_nom"] partial["pl_nom"] = partial["nt_sg_nom"] @@ -65,82 +53,63 @@ def complete_paradigm(partial, complete=False): if complete: for gender in ["mi", "ma", "mp", "nt", "f"]: for case in ["nom", "gen", "dat", "acc", "ins", "loc", "voc"]: - key = f'{gender}_pl_{case}' + key = f"{gender}_pl_{case}" if key not in partial: - partial[key] = partial[f'pl_{case}'] + partial[key] = partial[f"pl_{case}"] -def make_graph_dict(filepath, invert=True, complete=False): - output_graph = {} - word_tsv = load_labels(get_abs_path(filepath)) - for word, target in word_tsv: - word_forms = adjective_inflection(word) +def make_graph_dict(filepath: str, invert: bool = True, complete: bool = False): + output = {} + for word, target in load_labels(get_abs_path(filepath)): + forms = adjective_inflection(word) if complete: - complete_paradigm(word_forms, complete=True) - for key in word_forms: - if invert: - a = target - b = word_forms[key] - else: - a = word_forms[key] - b = target - if key not in output_graph: - output_graph[key] = pynini.cross(a, b) - else: - output_graph[key] |= pynini.cross(a, b) - return output_graph + complete_paradigm(forms, complete=True) + for slot, form in forms.items(): + source, destination = (target, form) if invert else (form, target) + graph = pynini.cross(source, destination) + output[slot] = graph if slot not in output else output[slot] | graph + return {slot: graph.optimize() for slot, graph in output.items()} class OrdinalFst(GraphFst): - """ - Finite state transducer for classifying cardinals, e.g. - "2." -> ordinal { integer: "drugi" } } - "2-gi" -> ordinal { integer: "drugi" } } - "123." -> ordinal { integer: "sto dwudziesty trzeci" } } - - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + """Classifies Polish ordinals and exposes adjective-inflected graphs in ``graphs``.""" - def __init__(self, deterministic=False): + def __init__(self, deterministic: bool = True): super().__init__(name="ordinal", kind="classify", deterministic=deterministic) - self.digits_all = make_graph_dict("data/ordinal/digit.tsv") - self.tens_all = make_graph_dict("data/ordinal/tens.tsv") - self.teens_all = make_graph_dict("data/ordinal/teens.tsv") - self.hundreds_all = make_graph_dict("data/ordinal/hundreds.tsv") - two_digit_all = self.make_two_digit() - - digits_graph = all_to_graph(self.digits_all, deterministic=deterministic) - tens_graph = all_to_graph(self.tens_all, deterministic=deterministic) - teens_graph = all_to_graph(self.teens_all, deterministic=deterministic) - hundreds_graph = all_to_graph(self.hundreds_all, deterministic=deterministic) - two_digit_graph = all_to_graph(two_digit_all, deterministic=deterministic) + digits = make_graph_dict("data/ordinal/digit.tsv", complete=True) + tens = make_graph_dict("data/ordinal/tens.tsv", complete=True) + teens = make_graph_dict("data/ordinal/teens.tsv", complete=True) + hundreds = make_graph_dict("data/ordinal/hundreds.tsv", complete=True) + cardinal_hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))).optimize() - wieczny_forms = adjective_inflection("wieczny") - wieczny_acceptor = pynini.union(*wieczny_forms.values()) + self.graphs = {} + for slot in digits: + joiner = pynutil.insert("") if slot == "compound" else insert_space + if slot == "compound" and not deterministic: + joiner |= pynutil.add_weight(insert_space, 0.001) - # self.graph = ( - # ( - # pynini.closure(NEMO_DIGIT | pynini.accep(".")) - # + pynutil.delete(pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")) - # ) - # @ cardinal_graph - # ).optimize() - # final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") - # final_graph = self.add_tokens(final_graph) - # self.fst = final_graph.optimize() - - def make_two_digit(self): - two_digits = {} - for key in self.digits_all: - two_digits[key] = self.tens_all[key] + pynutil.delete('0') - two_digits[key] |= pynutil.delete('0') + self.digits_all[key] - two_digits[key] |= self.teens_all[key] - if key != "compound": - two_digits[key] |= self.tens_all[key] + insert_space + self.digits_all[key] - else: - two_digits[key] |= self.tens_all[key] + self.digits_all[key] - return two_digits + two_digit = ( + tens[slot] + pynutil.delete("0") + | pynutil.delete("0") + digits[slot] + | teens[slot] + | tens[slot] + joiner + digits[slot] + ).optimize() + three_digit = ( + hundreds[slot] + pynutil.delete("00") + | pynutil.delete("0") + two_digit + | cardinal_hundreds + joiner + two_digit + ).optimize() + short_input = pynini.closure(NEMO_DIGIT, 1, 3) + pad = short_input @ pynini.cdrewrite( + pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA + ) @ NEMO_DIGIT**3 + self.graphs[slot] = (pad @ three_digit).optimize() + self.graph_dict = self.graphs + graph = all_to_graph(self.graphs, deterministic=deterministic) + if not deterministic: + graph = pynini.union(*self.graphs.values()).optimize() + self.graph = (graph + pynutil.delete(".")).optimize() + final_graph = pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/roman.py b/nemo_text_processing/text_normalization/pl/taggers/roman.py new file mode 100644 index 000000000..780c7e873 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/roman.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, delete_space, insert_space +from nemo_text_processing.text_normalization.pl.graph_utils import roman_to_int +from nemo_text_processing.text_normalization.pl.inflection import case_prepositions, inflect_noun +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +def _name_forms(name: str, grammar_files: str): + components = name.split(" ") + grammars = grammar_files.split(",") + if len(components) != len(grammars): + raise ValueError(f"{name!r} must have one grammar per component") + paradigms = [inflect_noun(component, grammar) for component, grammar in zip(components, grammars)] + slots = set.intersection(*(set(paradigm) for paradigm in paradigms)) + return {slot: " ".join(paradigm[slot] for paradigm in paradigms) for slot in slots} + + +class RomanFst(GraphFst): + """Classifies Roman ordinals following curated ruler and papal names.""" + + def __init__(self, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + prepositions = case_prepositions() + self.graphs = {} + for category, name, grammar_files in load_labels(get_abs_path("data/roman/names.tsv")): + gender = "f" if category == "queen" else "mp" + for noun_slot, surface_name in _name_forms(name, grammar_files).items(): + number, case = noun_slot.split("_", 1) + ordinal_slot = f"{gender}_{number}_{case}" + if ordinal_slot not in ordinal.graphs: + continue + name_graph = pynini.accep(surface_name) + if category == "pope" and noun_slot == "sg_nom": + title = pynini.union("Papież", "papież") + delete_space + insert_space + name_graph |= title + pynini.accep(surface_name) + graph = ( + name_graph + + delete_space + + insert_space + + roman_to_int(ordinal.graphs[ordinal_slot]) + ) + if case in prepositions: + graph |= prepositions[case] + graph + self.graphs[ordinal_slot] = ( + graph if ordinal_slot not in self.graphs else self.graphs[ordinal_slot] | graph + ) + + self.graph_dict = {slot: graph.optimize() for slot, graph in self.graphs.items()} + graph = pynini.union(*self.graph_dict.values()).optimize() + self.graph = graph + final_graph = pynutil.insert('name: "') + convert_space(graph) + pynutil.insert('"') + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/time.py b/nemo_text_processing/text_normalization/pl/taggers/time.py new file mode 100644 index 000000000..b52be28f3 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/time.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """Classifies Polish numeric hours and minutes.""" + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + hour_numbers = pynini.union(*(str(hour) for hour in range(1, 24))) + hours = hour_numbers | pynutil.delete("0") + pynini.union( + *(str(hour) for hour in range(1, 10)) + ) + minutes = pynini.union(*(f"{minute:02d}" for minute in range(1, 60))) + minute_words = ( + pynutil.delete("0") + cardinal.graphs["mi_sg_nom"] + ) | cardinal.graphs["mi_sg_nom"] + + def time_graph(hour_slot: str, prefix: 'pynini.FstLike') -> 'pynini.FstLike': + hour = hours @ ordinal.graphs[hour_slot] + hour_field = pynutil.insert('hours: "') + prefix + hour + pynutil.insert('"') + minute_field = pynutil.insert(' minutes: "') + (minutes @ minute_words) + pynutil.insert('"') + separator = pynutil.delete(pynini.union(":", ".")) + return hour_field + separator + (pynutil.delete("00") | minute_field) + + plain = time_graph("f_sg_nom", pynini.accep("")) + governed = time_graph("f_sg_loc", pynini.accep("o") + delete_space + pynutil.insert(" ")) + hour_abbreviation = ( + pynini.accep("o") + + delete_space + + pynutil.insert(" ") + + pynini.cross("godz.", "godzinie") + + delete_space + + pynutil.insert(" ") + ) + governed |= time_graph("f_sg_loc", hour_abbreviation) + + locale_hour = pynini.cross("00", "zero") | hours @ ordinal.graphs["f_sg_nom"] + locale_minute = pynini.cross("00", "zero") | minutes @ minute_words + locale_second = pynini.cross("00", "zero") | minutes @ minute_words + locale_time = ( + pynutil.insert('hours: "') + + locale_hour + + pynutil.insert('"') + + pynutil.delete(":") + + pynutil.insert(' minutes: "') + + locale_minute + + pynutil.insert('"') + + pynutil.delete(":") + + pynutil.insert(' seconds: "') + + locale_second + + pynutil.insert('"') + ) + self.final_graph = (plain | governed | locale_time).optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..df8632b19 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.pl.taggers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.taggers.date import DateFst +from nemo_text_processing.text_normalization.pl.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.pl.taggers.time import TimeFst +from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.pl.taggers.word import WordFst +from nemo_text_processing.utils.logging import logger +from pynini.lib import pynutil + + +class ClassifyFst(GraphFst): + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"pl_tn_{deterministic}_{input_case}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + return + + self.cardinal = CardinalFst(deterministic=deterministic) + self.ordinal = OrdinalFst(deterministic=deterministic) + self.roman = RomanFst(self.ordinal, deterministic=deterministic) + self.date = DateFst(self.cardinal, self.ordinal, deterministic=deterministic) + self.measure = MeasureFst(self.cardinal, deterministic=deterministic) + self.time = TimeFst(self.cardinal, self.ordinal, deterministic=deterministic) + self.whitelist = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ) + word = WordFst(deterministic=deterministic).fst + punctuation = PunctuationFst(deterministic=deterministic).fst + classify = ( + pynutil.add_weight(self.whitelist.fst, 1.01) + | pynutil.add_weight(self.roman.fst, 1.02) + | pynutil.add_weight(self.date.fst, 1.05) + | pynutil.add_weight(self.time.fst, 1.05) + | pynutil.add_weight(self.measure.fst, 1.06) + | pynutil.add_weight(self.ordinal.fst, 1.09) + | pynutil.add_weight(self.cardinal.fst, 1.1) + | pynutil.add_weight(punctuation, 2.1) + | pynutil.add_weight(word, 100) + ) + if not deterministic: + classify |= pynutil.add_weight( + AbbreviationFst(whitelist=self.whitelist, deterministic=False).fst, 100 + ) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + graph = delete_space + token + pynini.closure(delete_extra_space + token) + delete_space + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammar was saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py new file mode 100644 index 000000000..686f32869 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py @@ -0,0 +1,153 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Optional + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst as vAbbreviationFst +from nemo_text_processing.text_normalization.pl.taggers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.taggers.date import DateFst +from nemo_text_processing.text_normalization.pl.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.pl.taggers.time import TimeFst +from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.pl.verbalizers.cardinal import CardinalFst as vCardinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.date import DateFst as vDateFst +from nemo_text_processing.text_normalization.pl.verbalizers.measure import MeasureFst as vMeasureFst +from nemo_text_processing.text_normalization.pl.verbalizers.ordinal import OrdinalFst as vOrdinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.roman import RomanFst as vRomanFst +from nemo_text_processing.text_normalization.pl.verbalizers.time import TimeFst as vTimeFst +from pynini.lib import pynutil + + +class ClassifyFst(GraphFst): + """Composes Polish classification and verbalization for audio-based TN.""" + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = True, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"_{input_case}_pl_tn_{deterministic}_with_audio.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + else: + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + roman = RomanFst(ordinal, deterministic=deterministic) + date = DateFst(cardinal, ordinal, deterministic=deterministic) + measure = MeasureFst(cardinal, deterministic=deterministic) + time = TimeFst(cardinal, ordinal, deterministic=deterministic) + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ) + v_cardinal = vCardinalFst(deterministic=deterministic) + v_ordinal = vOrdinalFst(deterministic=deterministic) + v_roman = vRomanFst(deterministic=deterministic) + v_date = vDateFst(deterministic=deterministic) + v_measure = vMeasureFst(deterministic=deterministic) + v_time = vTimeFst(deterministic=deterministic) + word = pynini.closure(NEMO_NOT_SPACE, 1) + punctuation = PunctuationFst(deterministic=True).graph + + sem_w = 1 + word_w = 100 + punct_w = 2 + classify_and_verbalize = ( + pynutil.add_weight(whitelist_graph.graph, sem_w) + | pynutil.add_weight(pynini.compose(roman.fst, v_roman.fst), sem_w) + | pynutil.add_weight(pynini.compose(date.fst, v_date.fst), sem_w) + | pynutil.add_weight(pynini.compose(measure.fst, v_measure.fst), sem_w) + | pynutil.add_weight(pynini.compose(time.fst, v_time.fst), sem_w) + | pynutil.add_weight(pynini.compose(cardinal.fst, v_cardinal.fst), sem_w) + | pynutil.add_weight(pynini.compose(ordinal.fst, v_ordinal.fst), sem_w) + | pynutil.add_weight(word, word_w) + ).optimize() + if not deterministic: + abbreviation = AbbreviationFst(whitelist=whitelist_graph, deterministic=False) + v_abbreviation = vAbbreviationFst(deterministic=False) + classify_and_verbalize |= pynutil.add_weight( + pynini.compose(abbreviation.fst, v_abbreviation.fst), word_w + ) + punct_only = pynutil.add_weight(punctuation, punct_w) + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | pynutil.insert(" ") + punct_only, + 1, + ) + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + classify_and_verbalize + + pynini.closure(pynutil.insert(" ") + punct) + ) + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | pynutil.insert(" ") + punct + pynutil.insert(" ") + ) + + token_plus_punct + ) + graph |= punct_only + pynini.closure(punct) + graph = delete_space + graph + delete_space + remove_extra_spaces = pynini.closure(NEMO_NOT_SPACE, 1) + pynini.closure( + delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1) + ) + remove_extra_spaces |= ( + pynini.closure(pynutil.delete(" "), 1) + + pynini.closure(NEMO_NOT_SPACE, 1) + + pynini.closure(delete_extra_space + pynini.closure(NEMO_NOT_SPACE, 1)) + ) + self.fst = pynini.compose(graph.optimize(), remove_extra_spaces).optimize() + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + + def lattice(self, text: str) -> 'pynini.Fst': + lattice = pynini.compose(pynini.accep(pynini.escape(text)), self.fst) + if lattice.start() == pynini.NO_STATE_ID: + raise ValueError(f"Polish TN failed for input: {text}") + return lattice + + def normalize(self, text: str, lm: Optional['pynini.FstLike'] = None) -> str: + lattice = self.lattice(text) + if lm is not None: + lattice = pynini.compose(lattice, lm) + if lattice.start() == pynini.NO_STATE_ID: + raise ValueError("The language model rejected every Polish TN path") + best = pynini.shortestpath(lattice, nshortest=1, unique=True).project("output") + return best.string() diff --git a/nemo_text_processing/text_normalization/pl/taggers/whitelist.py b/nemo_text_processing/text_normalization/pl/taggers/whitelist.py new file mode 100644 index 000000000..fc1dbe590 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/whitelist.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from nemo_text_processing.text_normalization.pl.inflection import ( + load_adjective_abbreviations, + load_ambiguous_abbreviations, + load_inflected_abbreviations, +) +from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +def _get_whitelist_graph(input_case: str, filepath: str) -> 'pynini.FstLike': + labels = load_labels(filepath) + if input_case == "lower_cased": + labels = [[entry[0].lower()] + entry[1:] for entry in labels] + return pynini.string_map(labels).optimize() + + +class WhiteListFst(GraphFst): + """Classifies fixed and productively inflected Polish abbreviations.""" + + def __init__( + self, input_case: str, deterministic: bool = True, input_file: str = None + ): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), 0.0001 + ) + + self.inflected_graphs = load_inflected_abbreviations("data/abbreviations.tsv") + graph |= pynini.union(*self.inflected_graphs.values()) + + self.nondeterministic_graphs = load_ambiguous_abbreviations( + "data/abbreviations_nondet.tsv" + ) + self.adjective_graphs = load_adjective_abbreviations( + "data/abbreviations_adjective_nondet.tsv" + ) + if not deterministic: + graph |= pynini.union( + *self.nondeterministic_graphs.values(), *self.adjective_graphs.values() + ) + + if input_file: + provided = _get_whitelist_graph(input_case, input_file) + graph = graph | provided if not deterministic else provided + + self.graph = graph.optimize() + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert('name: "') + self.final_graph + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/word.py b/nemo_text_processing/text_normalization/pl/taggers/word.py new file mode 100644 index 000000000..6d8852ee0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/taggers/word.py @@ -0,0 +1,24 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst +from pynini.lib import pynutil + + +class WordFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + self.fst = ( + pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') + ).optimize() diff --git a/nemo_text_processing/text_normalization/pl/utils.py b/nemo_text_processing/text_normalization/pl/utils.py index a6d5f34dd..bbf1cdaa5 100644 --- a/nemo_text_processing/text_normalization/pl/utils.py +++ b/nemo_text_processing/text_normalization/pl/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -77,6 +77,11 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): mi_sg = word mp_pl = word vowel = "" + elif word.endswith("ony"): + stem = word[:-1] + mi_sg = word + mp_pl = word[:-3] + "eni" + vowel = "y" elif word.endswith("szy"): stem = word[:-1] mi_sg = word @@ -88,6 +93,12 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): mi_sg = word mp_pl = word[:-2] + "dzy" vowel = "" + elif word.endswith("ki"): + stem = word + stem_b = word[:-1] + mi_sg = word + mp_pl = word[:-2] + "cy" + vowel = "" elif word.endswith("sty"): stem = word[:-1] mi_sg = word diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/__init__.py b/nemo_text_processing/text_normalization/pl/verbalizers/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py new file mode 100644 index 000000000..815f64f64 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class CardinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + sign = pynini.closure(pynini.cross('negative: "true"', "minus") + delete_space, 0, 1) + integer = pynutil.delete("integer:") + delete_space + pynutil.delete('"') + integer += pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + self.fst = self.delete_tokens(sign + integer).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/date.py b/nemo_text_processing/text_normalization/pl/verbalizers/date.py new file mode 100644 index 000000000..e4e52acc9 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/date.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class DateFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + value = pynini.closure(NEMO_NOT_QUOTE, 1) + day = pynutil.delete('day: "') + value + pynutil.delete('"') + month = pynutil.delete('month: "') + value + pynutil.delete('"') + year = pynutil.delete('year: "') + value + pynutil.delete('"') + graph = day + delete_space + pynutil.insert(" ") + month + graph += pynini.closure(delete_space + pynutil.insert(" ") + year, 0, 1) + self.fst = self.delete_tokens(graph | year).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/measure.py b/nemo_text_processing/text_normalization/pl/verbalizers/measure.py new file mode 100644 index 000000000..34000e2b8 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/measure.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + value = pynini.closure(NEMO_NOT_QUOTE, 1) + cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + pynutil.delete('integer: "') + + value + + pynutil.delete('"') + + delete_space + + pynutil.delete("}") + ) + units = pynutil.delete('units: "') + value + pynutil.delete('"') + self.fst = self.delete_tokens(cardinal + delete_space + pynutil.insert(" ") + units).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py new file mode 100644 index 000000000..d6849e650 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py @@ -0,0 +1,24 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class OrdinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + integer = pynutil.delete("integer:") + delete_space + pynutil.delete('"') + integer += pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + self.fst = self.delete_tokens(integer).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/roman.py b/nemo_text_processing/text_normalization/pl/verbalizers/roman.py new file mode 100644 index 000000000..8c252e111 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/roman.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class RomanFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + name = pynutil.delete("name:") + delete_space + pynutil.delete('"') + name += pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + self.fst = self.delete_tokens(name).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/time.py b/nemo_text_processing/text_normalization/pl/verbalizers/time.py new file mode 100644 index 000000000..58e99a5af --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/time.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + value = pynini.closure(NEMO_NOT_QUOTE, 1) + hours = pynutil.delete('hours: "') + value + pynutil.delete('"') + minutes = pynini.closure( + delete_space + + pynutil.insert(" ") + + pynutil.delete('minutes: "') + + value + + pynutil.delete('"'), + 0, + 1, + ) + seconds = pynini.closure( + delete_space + + pynutil.insert(" ") + + pynutil.delete('seconds: "') + + value + + pynutil.delete('"'), + 0, + 1, + ) + self.fst = self.delete_tokens(hours + minutes + seconds).optimize() diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py new file mode 100644 index 000000000..cb7020e00 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.pl.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.pl.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.pl.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.verbalizers.roman import RomanFst +from nemo_text_processing.text_normalization.pl.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + self.fst = ( + CardinalFst(deterministic=deterministic).fst + | OrdinalFst(deterministic=deterministic).fst + | RomanFst(deterministic=deterministic).fst + | DateFst(deterministic=deterministic).fst + | MeasureFst(deterministic=deterministic).fst + | TimeFst(deterministic=deterministic).fst + | AbbreviationFst(deterministic=deterministic).fst + ) diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py new file mode 100644 index 000000000..dbfb4e244 --- /dev/null +++ b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.pl.verbalizers.verbalize import VerbalizeFst +from pynini.lib import pynutil + + +class VerbalizeFinalFst(GraphFst): + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"pl_tn_{deterministic}_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + return + + types = VerbalizeFst(deterministic=deterministic).fst | WordFst(deterministic=deterministic).fst + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + self.fst = (delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space).optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/tests/nemo_text_processing/pl/__init__.py b/tests/nemo_text_processing/pl/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/tests/nemo_text_processing/pl/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/pl/data_text_normalization/__init__.py b/tests/nemo_text_processing/pl/data_text_normalization/__init__.py new file mode 100644 index 000000000..dbfe137c1 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..524b3d379 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,15 @@ +0~zero +1~jeden +2~dwa +11~jedenaście +21~dwadzieścia jeden +22~dwadzieścia dwa +105~sto pięć +999~dziewięćset dziewięćdziesiąt dziewięć +1000~tysiąc +2000~dwa tysiące +5000~pięć tysięcy +21000~dwadzieścia jeden tysięcy +1000000~milion +2000001~dwa miliony jeden +1832~tysiąc osiemset trzydzieści dwa diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..875701f16 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_date.txt @@ -0,0 +1,6 @@ +10 lutego~dziesiątego lutego +28 lutego 1832~dwudziestego ósmego lutego tysiąc osiemset trzydziestego drugiego +29.01.1991~dwudziestego dziewiątego stycznia tysiąc dziewięćset dziewięćdziesiątego pierwszego +7.VIII.2026~siódmego sierpnia dwa tysiące dwudziestego szóstego +29 listopada 1877r.~dwudziestego dziewiątego listopada tysiąc osiemset siedemdziesiątego siódmego roku +29 listopada 1877 r.~dwudziestego dziewiątego listopada tysiąc osiemset siedemdziesiątego siódmego roku diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..caff6405d --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,6 @@ +1 m~jeden metr +2 cm~dwa centymetry +16 Hz~szesnaście herców +20 kHz~dwadzieścia kiloherców +35cm~trzydzieści pięć centymetrów +1000 mm~tysiąc milimetrów diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt new file mode 100644 index 000000000..7f72da5ad --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_normalize_with_audio.txt @@ -0,0 +1,27 @@ +~Mam 22 koty +Mam dwadzieścia dwa koty +Mam dwadzieścia dwie koty +Mam dwudziestu dwóch koty +~To był 21. test +To był dwudziesty pierwszy test +To był dwudziesta pierwsza test +To był dwudziestego pierwszego test +~tzw. +tak zwany +tak zwanego +tak zwana +tak zwaną +~ur. +urodzony +urodzona +urodzonego +urodzeni +~ang. +angielski +angielska +angielskiego +angielscy +~z ang. +z angielskiego +~do 35 cm +do trzydziestu pięciu centymetrów diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..f9b59ce0a --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,9 @@ +1.~pierwszy +2.~drugi +11.~jedenasty +20.~dwudziesty +21.~dwudziesty pierwszy +100.~setny +101.~sto pierwszy +121.~sto dwudziesty pierwszy +999.~dziewięćset dziewięćdziesiąty dziewiąty diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..c86a57bc4 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,18 @@ +Ludwik XIV~Ludwik czternasty +Ludwika XIII~Ludwika trzynastego +o Ludwiku XVIII~o Ludwiku osiemnastym +dla Ludwika XV~dla Ludwika piętnastego +z Karolem V~z Karolem piątym +przez Ludwika XIV~przez Ludwika czternastego +Ludwikiem XIV~Ludwikiem czternastym +Henryka II~Henryka drugiego +Karol X~Karol dziesiąty +Karola X~Karola dziesiątego +Karolem V~Karolem piątym +Jerzy IV~Jerzy czwarty +Napoleona III~Napoleona trzeciego +Katarzyna II~Katarzyna druga +Papież Klemens VII~Papież Klemens siódmy +o Klemensie VII~o Klemensie siódmym +Franciszka Józefa I~Franciszka Józefa pierwszego +Jana Pawła II~Jana Pawła drugiego diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..cc0716b8d --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_time.txt @@ -0,0 +1,5 @@ +o 4.25~o czwartej dwadzieścia pięć +o godz. 21:14~o godzinie dwudziestej pierwszej czternaście +4:25~czwarta dwadzieścia pięć +04:25:07~czwarta dwadzieścia pięć siedem +00:00:00~zero zero zero diff --git a/tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..7e33c1681 --- /dev/null +++ b/tests/nemo_text_processing/pl/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,12 @@ +n.p.m.~nad poziomem morza +tj.~to jest +np.~na przykład +itd.~i tak dalej +m.in.~między innymi +im.~imienia +s-ka~spółka +s-ki~spółki +s-ce~spółce +ur.~urodzony +ang.~angielski +tzw.~tak zwany diff --git a/tests/nemo_text_processing/pl/test_abbreviation.py b/tests/nemo_text_processing/pl/test_abbreviation.py new file mode 100644 index 000000000..8f9664f69 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_abbreviation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from nemo_text_processing.text_normalization.pl.taggers.abbreviation import AbbreviationFst + + +class TestAbbreviation: + normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_graph(self): + abbreviation = AbbreviationFst(deterministic=False) + assert 'abbreviation { value: "A B C" }' in rewrite.top_rewrites("ABC", abbreviation.fst, 10) + assert 'abbreviation { value: "A. B. C." }' in rewrite.top_rewrites("A.B.C.", abbreviation.fst, 10) + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_audio_lattice(self): + predictions = self.normalizer.normalize("Kod ABC", n_tagged=20, punct_post_process=False) + assert "Kod A B C" in predictions diff --git a/tests/nemo_text_processing/pl/test_audio_normalizer.py b/tests/nemo_text_processing/pl/test_audio_normalizer.py new file mode 100644 index 000000000..f4a19fdb9 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_audio_normalizer.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +import pytest + +from nemo_text_processing.text_normalization.pl.taggers.tokenize_and_classify_with_audio import ClassifyFst + + +class TestAudioNormalizer: + normalizer = ClassifyFst(input_case="cased") + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_ngram_acceptor_selects_inflection(self): + masculine = pynini.accep("Mam dwadzieścia dwa koty", weight=1) + feminine = pynini.accep("Mam dwadzieścia dwie koty", weight=0) + language_model = (masculine | feminine).optimize() + assert self.normalizer.normalize("Mam 22 koty", language_model) == "Mam dwadzieścia dwie koty" + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_ngram_acceptor_selects_compound_tokenization(self): + joined = pynini.accep("To dwudziestodwulatka", weight=1) + spaced = pynini.accep("To dwudziesto dwu latka", weight=0) + language_model = (joined | spaced).optimize() + assert self.normalizer.normalize("To 22-latka", language_model) == "To dwudziesto dwu latka" diff --git a/tests/nemo_text_processing/pl/test_cardinal.py b/tests/nemo_text_processing/pl/test_cardinal.py new file mode 100644 index 000000000..5884e044c --- /dev/null +++ b/tests/nemo_text_processing/pl/test_cardinal.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst + +from ..utils import parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_cardinal.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_inflectional_graphs(self): + cardinal = CardinalFst() + assert rewrite.one_top_rewrite("1", cardinal.graphs["f_sg_nom"]) == "jedna" + assert rewrite.one_top_rewrite("2", cardinal.graphs["mp_pl_nom"]) == "dwaj" + assert rewrite.one_top_rewrite("22", cardinal.graphs["mp_pl_nom"]) == "dwudziestu dwóch" + assert rewrite.one_top_rewrite("22", cardinal.graphs["f_pl_nom"]) == "dwadzieścia dwie" + assert rewrite.one_top_rewrite("22", cardinal.graphs["pl_gen"]) == "dwudziestu dwóch" + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_compound_graph(self): + cardinal = CardinalFst() + assert rewrite.one_top_rewrite("22", cardinal.graphs["compound"]) == "dwudziestodwu" + assert rewrite.one_top_rewrite("22-latka", cardinal.compound) == "dwudziestodwulatka" + + cardinal = CardinalFst(deterministic=False) + alternatives = rewrite.top_rewrites("22", cardinal.graphs["compound"], 10) + assert "dwudziestodwu" in alternatives + assert "dwudziesto dwu" in alternatives + alternatives = rewrite.top_rewrites("22-latka", cardinal.compound, 20) + assert "dwudziestodwulatka" in alternatives + assert "dwudziesto dwu latka" in alternatives diff --git a/tests/nemo_text_processing/pl/test_date.py b/tests/nemo_text_processing/pl/test_date.py new file mode 100644 index 000000000..8ff568ef1 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_date.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pl.taggers.date import DateFst +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst + +from ..utils import parse_test_case_file + + +class TestDate: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_date.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_case_coordinated_graphs(self): + cardinal = CardinalFst(deterministic=True) + ordinal = OrdinalFst(deterministic=True) + date = DateFst(cardinal, ordinal, deterministic=True) + assert "mi_sg_nom" in date.graphs + assert "mi_sg_gen" in date.graphs + assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_nom"]) == ( + 'day: "dziesiąty" month: "lutego"' + ) + assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_gen"]) == ( + 'day: "dziesiątego" month: "lutego"' + ) + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_bare_year_is_only_a_nondeterministic_date(self): + cardinal = CardinalFst(deterministic=False) + ordinal = OrdinalFst(deterministic=False) + date = DateFst(cardinal, ordinal, deterministic=False) + assert rewrite.one_top_rewrite("1832", date.year_graphs["mi_sg_gen"]) == ( + 'year: "tysiąc osiemset trzydziestego drugiego"' + ) diff --git a/tests/nemo_text_processing/pl/test_measure.py b/tests/nemo_text_processing/pl/test_measure.py new file mode 100644 index 000000000..834937d1e --- /dev/null +++ b/tests/nemo_text_processing/pl/test_measure.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import parse_test_case_file + + +class TestMeasure: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_measure.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected diff --git a/tests/nemo_text_processing/pl/test_normalization_with_audio.py b/tests/nemo_text_processing/pl/test_normalization_with_audio.py new file mode 100644 index 000000000..a8c26bffc --- /dev/null +++ b/tests/nemo_text_processing/pl/test_normalization_with_audio.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + +from ..utils import get_test_cases_multiple + + +class TestNormalizeWithAudio: + normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand( + get_test_cases_multiple("pl/data_text_normalization/test_cases_normalize_with_audio.txt") + ) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_normalization_with_audio(self, test_input, expected): + predictions = self.normalizer.normalize(test_input, n_tagged=100, punct_post_process=False) + for option in expected: + assert option in predictions diff --git a/tests/nemo_text_processing/pl/test_ordinal.py b/tests/nemo_text_processing/pl/test_ordinal.py new file mode 100644 index 000000000..dd356bc0f --- /dev/null +++ b/tests/nemo_text_processing/pl/test_ordinal.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst + +from ..utils import parse_test_case_file + + +class TestOrdinal: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_ordinal.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_inflectional_graphs(self): + ordinal = OrdinalFst() + assert rewrite.one_top_rewrite("21", ordinal.graphs["f_sg_nom"]) == "dwudziesta pierwsza" + assert rewrite.one_top_rewrite("21", ordinal.graphs["mi_sg_gen"]) == "dwudziestego pierwszego" diff --git a/tests/nemo_text_processing/pl/test_roman.py b/tests/nemo_text_processing/pl/test_roman.py new file mode 100644 index 000000000..7ccf22314 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_roman.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.pl.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pl.taggers.roman import RomanFst + +from ..utils import parse_test_case_file + + +class TestRoman: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_roman.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_two_component_name_inflection(self): + roman = RomanFst(OrdinalFst()) + assert rewrite.one_top_rewrite("Franciszka Józefa I", roman.graphs["mp_sg_gen"]) == ( + "Franciszka Józefa pierwszego" + ) diff --git a/tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..17870bc0c --- /dev/null +++ b/tests/nemo_text_processing/pl/test_sparrowhawk_normalization.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} + +runtest() { + input=$1 + echo "INPUT is $input" + cd "${GRAMMARS_DIR}" || return 1 + + while read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + denorm_pred=$(echo "$written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/ / /g' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_cardinal.txt" +} + +testTNDate() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_date.txt" +} + +testTNMeasure() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_measure.txt" +} + +testTNOrdinal() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_ordinal.txt" +} + +testTNRoman() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_roman.txt" +} + +testTNTime() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_time.txt" +} + +testTNWhitelist() { + runtest "$PROJECT_DIR/pl/data_text_normalization/test_cases_whitelist.txt" +} + +. "$PROJECT_DIR/../shunit2/shunit2" diff --git a/tests/nemo_text_processing/pl/test_time.py b/tests/nemo_text_processing/pl/test_time.py new file mode 100644 index 000000000..61797dc3f --- /dev/null +++ b/tests/nemo_text_processing/pl/test_time.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import parse_test_case_file + + +class TestTime: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_time.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected diff --git a/tests/nemo_text_processing/pl/test_whitelist.py b/tests/nemo_text_processing/pl/test_whitelist.py new file mode 100644 index 000000000..b2f29a221 --- /dev/null +++ b/tests/nemo_text_processing/pl/test_whitelist.py @@ -0,0 +1,61 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib import rewrite + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst + +from ..utils import parse_test_case_file + + +class TestWhitelist: + normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) + audio_normalizer = NormalizerWithAudio( + input_case="cased", lang="pl", cache_dir=None, post_process=False + ) + + @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_whitelist.txt")) + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_norm(self, test_input, expected): + prediction = self.normalizer.normalize(test_input, punct_post_process=False) + assert prediction == expected + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_inflected_graphs_are_keyed_by_slot(self): + whitelist = WhiteListFst(input_case="cased", deterministic=True) + assert rewrite.one_top_rewrite("s-ka", whitelist.inflected_graphs["sg_nom"]) == "spółka" + assert rewrite.one_top_rewrite("s-ki", whitelist.inflected_graphs["sg_gen"]) == "spółki" + assert rewrite.one_top_rewrite("s-ce", whitelist.inflected_graphs["sg_loc"]) == "spółce" + + @pytest.mark.run_only_on("CPU") + @pytest.mark.unit + def test_ambiguous_entry_has_all_singular_forms(self): + whitelist = WhiteListFst(input_case="cased", deterministic=False) + expected = { + "sg_nom": "rok", + "sg_gen": "roku", + "sg_dat": "rokowi", + "sg_acc": "rok", + "sg_ins": "rokiem", + "sg_loc": "roku", + "sg_voc": "roku", + } + for slot, spoken in expected.items(): + assert rewrite.one_top_rewrite("r.", whitelist.nondeterministic_graphs[slot]) == spoken diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..e55704666 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -95,6 +95,7 @@ def parse_args(): "ru", 'fr', 'hu', + 'pl', 'sv', 'vi', 'zh', @@ -240,6 +241,11 @@ def parse_args(): ClassifyFst as TNClassifyFst, ) from nemo_text_processing.text_normalization.sv.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'pl': + from nemo_text_processing.text_normalization.pl.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.pl.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'vi': from nemo_text_processing.inverse_text_normalization.vi.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, From 83f1af2480c2c8a68a758c1472e10822db7d92c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:52:44 +0000 Subject: [PATCH 73/73] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/pl/graph_utils.py | 3 ++- .../text_normalization/pl/inflection.py | 7 +++-- .../pl/taggers/abbreviation.py | 3 ++- .../text_normalization/pl/taggers/cardinal.py | 24 +++++++---------- .../text_normalization/pl/taggers/date.py | 27 +++++-------------- .../text_normalization/pl/taggers/measure.py | 3 ++- .../text_normalization/pl/taggers/ordinal.py | 11 +++++--- .../text_normalization/pl/taggers/roman.py | 10 +++---- .../text_normalization/pl/taggers/time.py | 11 +++----- .../pl/taggers/tokenize_and_classify.py | 11 +++----- .../tokenize_and_classify_with_audio.py | 7 +++-- .../pl/taggers/whitelist.py | 19 +++++-------- .../text_normalization/pl/taggers/word.py | 7 +++-- .../text_normalization/pl/utils.py | 4 ++- .../pl/verbalizers/cardinal.py | 3 ++- .../text_normalization/pl/verbalizers/date.py | 3 ++- .../pl/verbalizers/measure.py | 3 ++- .../pl/verbalizers/ordinal.py | 3 ++- .../pl/verbalizers/roman.py | 3 ++- .../text_normalization/pl/verbalizers/time.py | 15 +++-------- .../pl/verbalizers/verbalize_final.py | 3 ++- tests/nemo_text_processing/pl/test_date.py | 8 ++---- .../pl/test_normalization_with_audio.py | 4 +-- .../nemo_text_processing/pl/test_whitelist.py | 4 +-- 24 files changed, 79 insertions(+), 117 deletions(-) diff --git a/nemo_text_processing/text_normalization/pl/graph_utils.py b/nemo_text_processing/text_normalization/pl/graph_utils.py index 1a7aa2387..cbad45062 100644 --- a/nemo_text_processing/text_normalization/pl/graph_utils.py +++ b/nemo_text_processing/text_normalization/pl/graph_utils.py @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space from pynini.lib import byte, pynutil +from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space + from .utils import get_abs_path, load_labels _ALPHA_UPPER = "AĄBCĆDEĘFGHIJKLŁMNŃOÓPQRSŚTUVWXYZŻŹ" diff --git a/nemo_text_processing/text_normalization/pl/inflection.py b/nemo_text_processing/text_normalization/pl/inflection.py index aa14a3715..7b1076577 100644 --- a/nemo_text_processing/text_normalization/pl/inflection.py +++ b/nemo_text_processing/text_normalization/pl/inflection.py @@ -15,9 +15,10 @@ from typing import Dict import pynini -from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels from pynini.lib import pynutil +from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels + def _load_endings(grammar_file: str) -> Dict[str, str]: return { @@ -54,9 +55,7 @@ def inflect_abbreviation(abbreviation: str, word: str, grammar_file: str) -> Dic endings = _load_endings(grammar_file) lemma_ending = endings["sg_nom"] if not abbreviation.endswith(lemma_ending) or not word.endswith(lemma_ending): - raise ValueError( - f"{abbreviation!r} and {word!r} must share the {lemma_ending!r} ending from {grammar_file}" - ) + raise ValueError(f"{abbreviation!r} and {word!r} must share the {lemma_ending!r} ending from {grammar_file}") abbreviation_stem = abbreviation[: -len(lemma_ending)] if lemma_ending else abbreviation word_stem = word[: -len(lemma_ending)] if lemma_ending else word return { diff --git a/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py b/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py index 382da8319..9cc8eac7d 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py +++ b/nemo_text_processing/text_normalization/pl/taggers/abbreviation.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space from nemo_text_processing.text_normalization.pl.graph_utils import PL_UPPER -from pynini.lib import pynutil class AbbreviationFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py index 6d22d2fff..5467262a9 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/cardinal.py @@ -15,11 +15,11 @@ from typing import Dict, Iterable import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, delete_space from nemo_text_processing.text_normalization.pl.graph_utils import PL_ALPHA from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels -from pynini.lib import pynutil - CASES = ["nom", "gen", "dat", "acc", "ins", "loc", "voc"] DEFAULT_SLOT = "mi_sg_nom" @@ -212,12 +212,8 @@ def __init__(self, deterministic: bool = True): self.graph_dict = self.graphs compound_boundary = pynutil.delete("-") if not deterministic: - compound_boundary += pynini.union( - pynutil.insert(""), pynutil.add_weight(pynutil.insert(" "), 0.001) - ) - self.compound = ( - self.graphs["compound"] + compound_boundary + pynini.closure(PL_ALPHA, 1) - ).optimize() + compound_boundary += pynini.union(pynutil.insert(""), pynutil.add_weight(pynutil.insert(" "), 0.001)) + self.compound = (self.graphs["compound"] + compound_boundary + pynini.closure(PL_ALPHA, 1)).optimize() self.graph = filter_punctuation(self.graphs[DEFAULT_SLOT] | self.zero_all["sg_nom"]).optimize() | self.compound if not deterministic: @@ -279,9 +275,11 @@ def _make_full_number_graph(self, group, slot, deterministic): case = _case_for_slot(slot) if case == "compound": short_input = pynini.closure(NEMO_DIGIT, 1, 3) - pad = short_input @ pynini.cdrewrite( - pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA - ) @ NEMO_DIGIT**3 + pad = ( + short_input + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**3 + ) return (pad @ group).optimize() scale_slot = { @@ -318,9 +316,7 @@ def _make_full_number_graph(self, group, slot, deterministic): | non_one_group + pynutil.insert(" " + forms[f"pl_{case}"] + " ") ) if not deterministic: - factor |= pynutil.add_weight( - pynini.cross("001", "jeden " + forms[f"sg_{case}"] + " "), 0.001 - ) + factor |= pynutil.add_weight(pynini.cross("001", "jeden " + forms[f"sg_{case}"] + " "), 0.001) factors.append(factor) padded = ( diff --git a/nemo_text_processing/text_normalization/pl/taggers/date.py b/nemo_text_processing/text_normalization/pl/taggers/date.py index 33ad30e78..79678f0b0 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/date.py +++ b/nemo_text_processing/text_normalization/pl/taggers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space from nemo_text_processing.text_normalization.pl.utils import get_abs_path -from pynini.lib import pynutil class DateFst(GraphFst): @@ -33,15 +34,11 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = month_abbr = pynini.string_file(get_abs_path("data/dates/month_abbr.tsv")) month_roman = pynini.string_file(get_abs_path("data/dates/months_roman.tsv")) - year_prefix = ( - (NEMO_DIGIT - "0") + pynutil.insert("000") - ) @ cardinal.graphs["mi_sg_nom"] + year_prefix = ((NEMO_DIGIT - "0") + pynutil.insert("000")) @ cardinal.graphs["mi_sg_nom"] month_numeric_field = pynutil.insert(' month: "') + month_number + pynutil.insert('"') month_roman_field = pynutil.insert(' month: "') + month_roman + pynutil.insert('"') - month_word_field = ( - pynutil.insert(' month: "') + (month_words | month_abbr) + pynutil.insert('"') - ) + month_word_field = pynutil.insert(' month: "') + (month_words | month_abbr) + pynutil.insert('"') self.graphs = {} self.year_graphs = {} @@ -68,25 +65,15 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = for separator in (".", "-", "/") ) ) - numeric |= ( - numeric_day_field - + pynutil.delete(".") - + month_roman_field - + pynutil.delete(".") - + year_field - ) + numeric |= numeric_day_field + pynutil.delete(".") + month_roman_field + pynutil.delete(".") + year_field written = day_field + delete_space + month_word_field written += pynini.closure(delete_space + year_field, 0, 1) self.graphs[slot] = (numeric | written).optimize() - self.year_graphs[slot] = ( - pynutil.insert('year: "') + year + pynutil.insert('"') - ).optimize() + self.year_graphs[slot] = (pynutil.insert('year: "') + year + pynutil.insert('"')).optimize() self.graph_dict = self.graphs if deterministic: self.final_graph = self.graphs["mi_sg_gen"] else: - self.final_graph = pynini.union( - *self.graphs.values(), *self.year_graphs.values() - ).optimize() + self.final_graph = pynini.union(*self.graphs.values(), *self.year_graphs.values()).optimize() self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/measure.py b/nemo_text_processing/text_normalization/pl/taggers/measure.py index e8746bab4..95bc07ef0 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/measure.py +++ b/nemo_text_processing/text_normalization/pl/taggers/measure.py @@ -13,10 +13,11 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space from nemo_text_processing.text_normalization.pl.inflection import inflect_noun from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels -from pynini.lib import pynutil def _case(slot: str) -> str: diff --git a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py index 0b45d82ab..fc8c94007 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/taggers/ordinal.py @@ -14,10 +14,11 @@ from typing import Dict import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space from nemo_text_processing.text_normalization.pl.graph_utils import all_to_graph from nemo_text_processing.text_normalization.pl.utils import adjective_inflection, get_abs_path, load_labels -from pynini.lib import pynutil def complete_paradigm(partial: Dict[str, str], complete: bool = False): @@ -101,9 +102,11 @@ def __init__(self, deterministic: bool = True): | cardinal_hundreds + joiner + two_digit ).optimize() short_input = pynini.closure(NEMO_DIGIT, 1, 3) - pad = short_input @ pynini.cdrewrite( - pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA - ) @ NEMO_DIGIT**3 + pad = ( + short_input + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT**3 + ) self.graphs[slot] = (pad @ three_digit).optimize() self.graph_dict = self.graphs diff --git a/nemo_text_processing/text_normalization/pl/taggers/roman.py b/nemo_text_processing/text_normalization/pl/taggers/roman.py index 780c7e873..cd21d440f 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/roman.py +++ b/nemo_text_processing/text_normalization/pl/taggers/roman.py @@ -13,11 +13,12 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, delete_space, insert_space from nemo_text_processing.text_normalization.pl.graph_utils import roman_to_int from nemo_text_processing.text_normalization.pl.inflection import case_prepositions, inflect_noun from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels -from pynini.lib import pynutil def _name_forms(name: str, grammar_files: str): @@ -49,12 +50,7 @@ def __init__(self, ordinal: GraphFst, deterministic: bool = True): if category == "pope" and noun_slot == "sg_nom": title = pynini.union("Papież", "papież") + delete_space + insert_space name_graph |= title + pynini.accep(surface_name) - graph = ( - name_graph - + delete_space - + insert_space - + roman_to_int(ordinal.graphs[ordinal_slot]) - ) + graph = name_graph + delete_space + insert_space + roman_to_int(ordinal.graphs[ordinal_slot]) if case in prepositions: graph |= prepositions[case] + graph self.graphs[ordinal_slot] = ( diff --git a/nemo_text_processing/text_normalization/pl/taggers/time.py b/nemo_text_processing/text_normalization/pl/taggers/time.py index b52be28f3..90fff1de4 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/time.py +++ b/nemo_text_processing/text_normalization/pl/taggers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space + class TimeFst(GraphFst): """Classifies Polish numeric hours and minutes.""" @@ -24,13 +25,9 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = super().__init__(name="time", kind="classify", deterministic=deterministic) hour_numbers = pynini.union(*(str(hour) for hour in range(1, 24))) - hours = hour_numbers | pynutil.delete("0") + pynini.union( - *(str(hour) for hour in range(1, 10)) - ) + hours = hour_numbers | pynutil.delete("0") + pynini.union(*(str(hour) for hour in range(1, 10))) minutes = pynini.union(*(f"{minute:02d}" for minute in range(1, 60))) - minute_words = ( - pynutil.delete("0") + cardinal.graphs["mi_sg_nom"] - ) | cardinal.graphs["mi_sg_nom"] + minute_words = (pynutil.delete("0") + cardinal.graphs["mi_sg_nom"]) | cardinal.graphs["mi_sg_nom"] def time_graph(hour_slot: str, prefix: 'pynini.FstLike') -> 'pynini.FstLike': hour = hours @ ordinal.graphs[hour_slot] diff --git a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py index df8632b19..12b1c8ecd 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify.py @@ -14,6 +14,8 @@ import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -31,7 +33,6 @@ from nemo_text_processing.text_normalization.pl.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.pl.taggers.word import WordFst from nemo_text_processing.utils.logging import logger -from pynini.lib import pynutil class ClassifyFst(GraphFst): @@ -58,9 +59,7 @@ def __init__( self.date = DateFst(self.cardinal, self.ordinal, deterministic=deterministic) self.measure = MeasureFst(self.cardinal, deterministic=deterministic) self.time = TimeFst(self.cardinal, self.ordinal, deterministic=deterministic) - self.whitelist = WhiteListFst( - input_case=input_case, deterministic=deterministic, input_file=whitelist - ) + self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) word = WordFst(deterministic=deterministic).fst punctuation = PunctuationFst(deterministic=deterministic).fst classify = ( @@ -75,9 +74,7 @@ def __init__( | pynutil.add_weight(word, 100) ) if not deterministic: - classify |= pynutil.add_weight( - AbbreviationFst(whitelist=self.whitelist, deterministic=False).fst, 100 - ) + classify |= pynutil.add_weight(AbbreviationFst(whitelist=self.whitelist, deterministic=False).fst, 100) token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") graph = delete_space + token + pynini.closure(delete_extra_space + token) + delete_space self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py index 686f32869..cc4177c67 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py +++ b/nemo_text_processing/text_normalization/pl/taggers/tokenize_and_classify_with_audio.py @@ -16,6 +16,8 @@ from typing import Optional import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -42,7 +44,6 @@ from nemo_text_processing.text_normalization.pl.verbalizers.ordinal import OrdinalFst as vOrdinalFst from nemo_text_processing.text_normalization.pl.verbalizers.roman import RomanFst as vRomanFst from nemo_text_processing.text_normalization.pl.verbalizers.time import TimeFst as vTimeFst -from pynini.lib import pynutil class ClassifyFst(GraphFst): @@ -71,9 +72,7 @@ def __init__( date = DateFst(cardinal, ordinal, deterministic=deterministic) measure = MeasureFst(cardinal, deterministic=deterministic) time = TimeFst(cardinal, ordinal, deterministic=deterministic) - whitelist_graph = WhiteListFst( - input_case=input_case, deterministic=deterministic, input_file=whitelist - ) + whitelist_graph = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) v_cardinal = vCardinalFst(deterministic=deterministic) v_ordinal = vOrdinalFst(deterministic=deterministic) v_roman = vRomanFst(deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/pl/taggers/whitelist.py b/nemo_text_processing/text_normalization/pl/taggers/whitelist.py index fc1dbe590..7c68b4555 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/pl/taggers/whitelist.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.pl.inflection import ( load_adjective_abbreviations, @@ -20,7 +22,6 @@ load_inflected_abbreviations, ) from nemo_text_processing.text_normalization.pl.utils import get_abs_path, load_labels -from pynini.lib import pynutil def _get_whitelist_graph(input_case: str, filepath: str) -> 'pynini.FstLike': @@ -33,9 +34,7 @@ def _get_whitelist_graph(input_case: str, filepath: str) -> 'pynini.FstLike': class WhiteListFst(GraphFst): """Classifies fixed and productively inflected Polish abbreviations.""" - def __init__( - self, input_case: str, deterministic: bool = True, input_file: str = None - ): + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): super().__init__(name="whitelist", kind="classify", deterministic=deterministic) graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) @@ -47,16 +46,10 @@ def __init__( self.inflected_graphs = load_inflected_abbreviations("data/abbreviations.tsv") graph |= pynini.union(*self.inflected_graphs.values()) - self.nondeterministic_graphs = load_ambiguous_abbreviations( - "data/abbreviations_nondet.tsv" - ) - self.adjective_graphs = load_adjective_abbreviations( - "data/abbreviations_adjective_nondet.tsv" - ) + self.nondeterministic_graphs = load_ambiguous_abbreviations("data/abbreviations_nondet.tsv") + self.adjective_graphs = load_adjective_abbreviations("data/abbreviations_adjective_nondet.tsv") if not deterministic: - graph |= pynini.union( - *self.nondeterministic_graphs.values(), *self.adjective_graphs.values() - ) + graph |= pynini.union(*self.nondeterministic_graphs.values(), *self.adjective_graphs.values()) if input_file: provided = _get_whitelist_graph(input_case, input_file) diff --git a/nemo_text_processing/text_normalization/pl/taggers/word.py b/nemo_text_processing/text_normalization/pl/taggers/word.py index 6d8852ee0..9c0cdb7c7 100644 --- a/nemo_text_processing/text_normalization/pl/taggers/word.py +++ b/nemo_text_processing/text_normalization/pl/taggers/word.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="word", kind="classify", deterministic=deterministic) - self.fst = ( - pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') - ).optimize() + self.fst = (pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/pl/utils.py b/nemo_text_processing/text_normalization/pl/utils.py index bbf1cdaa5..6ba2a6e12 100644 --- a/nemo_text_processing/text_normalization/pl/utils.py +++ b/nemo_text_processing/text_normalization/pl/utils.py @@ -22,7 +22,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path @@ -47,6 +47,7 @@ def adjective_inflection(word: str, compound: str = "") -> dict: inflect adjectives based on their endings. This includes things like ordinals and 'jeden' (1) which inflect like adjectives. """ + def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): if stem_b == "": stem_b = stem @@ -66,6 +67,7 @@ def fill_bare_template(stem, mi_sg, mp_pl, vowel, stem_b="", compound=""): "pl_loc": stem + vowel + "ch", "compound": compound, } + stem_b = "" if word.endswith("en"): stem = word[:-2] + "n" diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py index 815f64f64..3120f7261 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/cardinal.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/date.py b/nemo_text_processing/text_normalization/pl/verbalizers/date.py index e4e52acc9..58ba637d4 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DateFst(GraphFst): def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/measure.py b/nemo_text_processing/text_normalization/pl/verbalizers/measure.py index 34000e2b8..ae4d6f13a 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class MeasureFst(GraphFst): def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py index d6849e650..b6f1f2960 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/ordinal.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/roman.py b/nemo_text_processing/text_normalization/pl/verbalizers/roman.py index 8c252e111..b844768d4 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/roman.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/roman.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class RomanFst(GraphFst): def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/time.py b/nemo_text_processing/text_normalization/pl/verbalizers/time.py index 58e99a5af..3a424be64 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class TimeFst(GraphFst): def __init__(self, deterministic: bool = True): @@ -23,20 +24,12 @@ def __init__(self, deterministic: bool = True): value = pynini.closure(NEMO_NOT_QUOTE, 1) hours = pynutil.delete('hours: "') + value + pynutil.delete('"') minutes = pynini.closure( - delete_space - + pynutil.insert(" ") - + pynutil.delete('minutes: "') - + value - + pynutil.delete('"'), + delete_space + pynutil.insert(" ") + pynutil.delete('minutes: "') + value + pynutil.delete('"'), 0, 1, ) seconds = pynini.closure( - delete_space - + pynutil.insert(" ") - + pynutil.delete('seconds: "') - + value - + pynutil.delete('"'), + delete_space + pynutil.insert(" ") + pynutil.delete('seconds: "') + value + pynutil.delete('"'), 0, 1, ) diff --git a/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py index dbfb4e244..04b2a403e 100644 --- a/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/pl/verbalizers/verbalize_final.py @@ -14,6 +14,8 @@ import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -22,7 +24,6 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.pl.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): diff --git a/tests/nemo_text_processing/pl/test_date.py b/tests/nemo_text_processing/pl/test_date.py index 8ff568ef1..2a65dc71f 100644 --- a/tests/nemo_text_processing/pl/test_date.py +++ b/tests/nemo_text_processing/pl/test_date.py @@ -42,12 +42,8 @@ def test_case_coordinated_graphs(self): date = DateFst(cardinal, ordinal, deterministic=True) assert "mi_sg_nom" in date.graphs assert "mi_sg_gen" in date.graphs - assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_nom"]) == ( - 'day: "dziesiąty" month: "lutego"' - ) - assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_gen"]) == ( - 'day: "dziesiątego" month: "lutego"' - ) + assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_nom"]) == ('day: "dziesiąty" month: "lutego"') + assert rewrite.one_top_rewrite("10 lutego", date.graphs["mi_sg_gen"]) == ('day: "dziesiątego" month: "lutego"') @pytest.mark.run_only_on("CPU") @pytest.mark.unit diff --git a/tests/nemo_text_processing/pl/test_normalization_with_audio.py b/tests/nemo_text_processing/pl/test_normalization_with_audio.py index a8c26bffc..ba9fb243d 100644 --- a/tests/nemo_text_processing/pl/test_normalization_with_audio.py +++ b/tests/nemo_text_processing/pl/test_normalization_with_audio.py @@ -22,9 +22,7 @@ class TestNormalizeWithAudio: normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) - @parameterized.expand( - get_test_cases_multiple("pl/data_text_normalization/test_cases_normalize_with_audio.txt") - ) + @parameterized.expand(get_test_cases_multiple("pl/data_text_normalization/test_cases_normalize_with_audio.txt")) @pytest.mark.run_only_on("CPU") @pytest.mark.unit def test_normalization_with_audio(self, test_input, expected): diff --git a/tests/nemo_text_processing/pl/test_whitelist.py b/tests/nemo_text_processing/pl/test_whitelist.py index b2f29a221..732d87aa1 100644 --- a/tests/nemo_text_processing/pl/test_whitelist.py +++ b/tests/nemo_text_processing/pl/test_whitelist.py @@ -25,9 +25,7 @@ class TestWhitelist: normalizer = Normalizer(input_case="cased", lang="pl", cache_dir=None, post_process=False) - audio_normalizer = NormalizerWithAudio( - input_case="cased", lang="pl", cache_dir=None, post_process=False - ) + audio_normalizer = NormalizerWithAudio(input_case="cased", lang="pl", cache_dir=None, post_process=False) @parameterized.expand(parse_test_case_file("pl/data_text_normalization/test_cases_whitelist.txt")) @pytest.mark.run_only_on("CPU")