diff --git a/README.md b/README.md index f4d3943..06375e4 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 | Hebrew | `he` | Yes | Yes | `hebrew` | | Hindi | `hi` | Yes | Yes | `hindi` | | Hindi–English (code-mixed) | `hi_en` | No | Yes | `hindi_english` | +| Hungarian | `hu` | Yes | Yes | `hungarian` | | Italian | `it` | Yes | Yes | `italian` | | Japanese | `ja` | Yes | Yes | `japanese` | | Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` | diff --git a/itn/hungarian/__init__.py b/itn/hungarian/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/itn/hungarian/data/dates/days.tsv b/itn/hungarian/data/dates/days.tsv new file mode 100644 index 0000000..f402f03 --- /dev/null +++ b/itn/hungarian/data/dates/days.tsv @@ -0,0 +1,31 @@ +1 elseje +2 másodika +3 harmadika +4 negyedike +5 ötödike +6 hatodika +7 hetedike +8 nyolcadika +9 kilencedike +10 tizedike +11 tizenegyedike +12 tizenkettedike +13 tizenharmadika +14 tizennegyedike +15 tizenötödike +16 tizenhatodika +17 tizenhetedike +18 tizennyolcadika +19 tizenkilencedike +20 huszadika +21 huszonegyedike +22 huszonkettedike +23 huszonharmadika +24 huszonnegyedike +25 huszonötödike +26 huszonhatodika +27 huszonhetedike +28 huszonnyolcadika +29 huszonkilencedike +30 harmincadika +31 harmincegyedike diff --git a/itn/hungarian/data/dates/months.tsv b/itn/hungarian/data/dates/months.tsv new file mode 100644 index 0000000..47a659e --- /dev/null +++ b/itn/hungarian/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 január +2 február +3 március +4 április +5 május +6 június +7 július +8 augusztus +9 szeptember +10 október +11 november +12 december \ No newline at end of file diff --git a/itn/hungarian/data/electronic/domain.tsv b/itn/hungarian/data/electronic/domain.tsv new file mode 100644 index 0000000..8955be4 --- /dev/null +++ b/itn/hungarian/data/electronic/domain.tsv @@ -0,0 +1,15 @@ +.com pont com +.org pont org +.gov pont gov +.uk pont u k +.fr pont f r +.net pont net +.br pont b r +.in pont i n +.ru pont r u +.de pont d e +.it pont i t +.es pont e s +.mx pont m x +.us pont u s +.hu pont hu diff --git a/itn/hungarian/data/electronic/server_name.tsv b/itn/hungarian/data/electronic/server_name.tsv new file mode 100644 index 0000000..7bf1f2b --- /dev/null +++ b/itn/hungarian/data/electronic/server_name.tsv @@ -0,0 +1,15 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast +origo +index +blikk +startlap \ No newline at end of file diff --git a/itn/hungarian/data/electronic/symbols.tsv b/itn/hungarian/data/electronic/symbols.tsv new file mode 100644 index 0000000..ef44fdd --- /dev/null +++ b/itn/hungarian/data/electronic/symbols.tsv @@ -0,0 +1,23 @@ +. pont +- kötőjel +_ aláhúzásjel +! felkiáltójel +# kettőskereszt +$ dollárjel +% százalékjel +& ésjel +' aposztróf +* csillagjel ++ pluszjel +/ perjel +\ fordított perjel += egyenlőjel +? kérdőjel +^ kúpos ékezet +` tompa ékezet +{ nyitó kapcsos zárójel +| függőleges vonal +} berekesztő kapcsos zárójel +~ hullámvonal +, vessző +: kettőspont diff --git a/itn/hungarian/data/inflection/endings.tsv b/itn/hungarian/data/inflection/endings.tsv new file mode 100644 index 0000000..7d23e89 --- /dev/null +++ b/itn/hungarian/data/inflection/endings.tsv @@ -0,0 +1,51 @@ +a át ának ával áért ává áig aként ában án ánál ába ára ához ából áról ától +ab ot nak bal ért bá ig ként ban on nál ba ra hoz ból ról tól +ág ot nak gal ért gá ig ként ban on nál ba ra hoz ból ról tól +ak at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ák at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +alék ot nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ály lyt lynak llyal lyért llyá lyig lyként lyban lyon lynál lyba lyra lyhoz lyból lyról lytól +amm ot nak al ért á ig ként ban on nál ba ra hoz ból ról tól +am ot nak mal ért má ig ként ban on nál ba ra hoz ból ról tól +ány nyt nynak nnyal nyért nnyá nyig nyként nyban nyon nynál nyba nyra nyhoz nyból nyról nytól +árd ot nak dal ért dá ig ként ban on nál ba ra hoz ból ról tól +ár t nak ral ért rá ig ként ban on nál ba ra hoz ból ról tól +á t nak val ért vá ig ként ban n nál ba ra hoz ból ról tól +at ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +é ét ének ével éért évé éig éként ében én énél ébe ére éhez éből éről étől +ef et nek fel ért fé ig ként ben en nél be re hez ből ről től +ég et nek gel ért gé ig ként ben en nél be re hez ből ről től +ek et nek kel ért ké ig ként ben en nél be re hez ből ről től +ék et nek kel ért ké ig ként ben en nél be re hez ből ről től +ény nyt nynek nnyel nyért nnyé nyig nyként nyben nyen nynél nybe nyre nyhez nyből nyről nytől +erc et nek cel ért cé ig ként ben en nél be re hez ből ről től +er t nek rel ért ré ig ként ben en nél be re hez ből ről től +ertz et nek cel ért cé ig ként ben en nél be re hez ből ről től +est et nek tel ért té ig ként ben en nél be re hez ből ről től +es t nek sel ért sé ig ként ben en nél be re hez ből ről től +és t nek sel ért sé ig ként ben en nél be re hez ből ről től +et et nek tel ért té ig ként ben en nél be re hez ből ről től +é t nek vel ért vé ig ként ben n nél be re hez ből ről től +evek et nek kel ért ké ig ként ben en nél be re hez ből ről től +év evet évnek évvel évért évvé évig évként évben éven évnél évbe évre évhez évből évről évtől +int et nek tel ért té ig ként ben en nél be re hez ből ről től +lyok at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +méteres et t nek sel ért sé ig ként ben en nél be re hez ből ről től +nyek et nek kel ért ké ig ként ben en nél be re hez ből ről től +nyok at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ok at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ók at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ők et nek kel ért ké ig ként ben ön nél be re höz ből ről től +ók ot nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +on t nak nal ért ná ig ként ban on nál ba ra hoz ból ról tól +ont ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +orint ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +or t nak ral ért rá ig ként ban on nál ba ra hoz ból ról tól +ort ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +os at t nak sal ért sá ig ként ban on nál ba ra hoz ból ról tól +ó t nak val ért vá ig ként ban n nál ba ra hoz ból ról tól +ő t nek vel ért vé ig ként ben n nél be re höz ből ről től +um ot nak mal ért má ig ként ban on nál ba ra hoz ból ról tól +us t nak sal ért sá ig ként ban on nál ba ra hoz ból ról tól +utak at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +út utat útnak úttal útért úttá útig útként útban úton útnál útba útra úthoz útból útról úttól diff --git a/itn/hungarian/data/inflection/plural_endings.tsv b/itn/hungarian/data/inflection/plural_endings.tsv new file mode 100644 index 0000000..2c73ccc --- /dev/null +++ b/itn/hungarian/data/inflection/plural_endings.tsv @@ -0,0 +1,39 @@ +er ek +erc ek +ertz ek +amm ok +um ok +alék ok +óra ák +or ok +méteres ek +es ek +és ek +ég ek +ő k ek +orint ok +ók ok +on ok +a ák +ort ok +ont ok +ény nyek +ály lyok +ány nyok +et ek +est ek +ág ok +at ok +ab ok +am ok +év evek +ef ek +int ek +os ak +ó k +us ok +árd ok +ár ok +út utak +é k +á k diff --git a/itn/hungarian/data/inflection/word_endings.tsv b/itn/hungarian/data/inflection/word_endings.tsv new file mode 100644 index 0000000..1d49e04 --- /dev/null +++ b/itn/hungarian/data/inflection/word_endings.tsv @@ -0,0 +1 @@ +óra át ának ával áért ává áig aként ában án ánál ába ára ához ából áról ától akor diff --git a/itn/hungarian/data/measures/measurements.tsv b/itn/hungarian/data/measures/measurements.tsv new file mode 100644 index 0000000..62b2a47 --- /dev/null +++ b/itn/hungarian/data/measures/measurements.tsv @@ -0,0 +1,52 @@ +cm centiméter +ó óra +g gramm +kg kilogramm +μg mikrogramm +mg milligramm +ng nanogramm +pg pikogramm +fg femtogramm +ag attogramm +km² négyzetkilométer +km2 négyzetkilométer +L liter +l liter +dl deciliter +dL deciliter +cl centiliter +cL centiliter +ml milliliter +mL milliliter +μl mikroliter +μL mikroliter +nl nanoliter +nL nanoliter +fl femtoliter +fL femtoliter +m méter +dm deciméter +cm centiméter +mm milliméter +dkm dekaméter +hm hektométer +km kilométer +mrm miriaméter +µm mikrométer +nm nanométer +pm pikométer +fm femtométer +am attométer +zm zeptométer +ym yoktométer +m² négyzetméter +m2 négyzetméter +m³ köbméter +m3 köbméter +mph mérföld per óra +mi/h mérföld per óra +ml milliliter +mm milliméter +ms milliszekundum +% százalék +mp másodperc \ No newline at end of file diff --git a/itn/hungarian/data/money/currency.tsv b/itn/hungarian/data/money/currency.tsv new file mode 100644 index 0000000..0354dbc --- /dev/null +++ b/itn/hungarian/data/money/currency.tsv @@ -0,0 +1,11 @@ +SEK svéd korona +GBP angol font +£ font +USD amerikai dollár +US$ amerikai dollár +$ dollár +IR£ ír font +IEP ír font +NOK norvég korona +HUF magyar forint +Ft forint diff --git a/itn/hungarian/data/money/currency_minor.tsv b/itn/hungarian/data/money/currency_minor.tsv new file mode 100644 index 0000000..b222ea2 --- /dev/null +++ b/itn/hungarian/data/money/currency_minor.tsv @@ -0,0 +1,11 @@ +SEK öre +GBP penny +USD cent +US$ cent +$ cent +IR£ penny +£ penny +IEP penny +NOK öre +HUF fillér +Ft fillér diff --git a/itn/hungarian/data/number/digit.tsv b/itn/hungarian/data/number/digit.tsv new file mode 100644 index 0000000..4254fd0 --- /dev/null +++ b/itn/hungarian/data/number/digit.tsv @@ -0,0 +1,9 @@ +egy 1 +kettő 2 +három 3 +négy 4 +öt 5 +hat 6 +hét 7 +nyolc 8 +kilenc 9 diff --git a/itn/hungarian/data/number/digit_inline.tsv b/itn/hungarian/data/number/digit_inline.tsv new file mode 100644 index 0000000..2336ab3 --- /dev/null +++ b/itn/hungarian/data/number/digit_inline.tsv @@ -0,0 +1,9 @@ +egy 1 +két 2 +három 3 +négy 4 +öt 5 +hat 6 +hét 7 +nyolc 8 +kilenc 9 diff --git a/itn/hungarian/data/number/quantities.tsv b/itn/hungarian/data/number/quantities.tsv new file mode 100644 index 0000000..7229518 --- /dev/null +++ b/itn/hungarian/data/number/quantities.tsv @@ -0,0 +1,21 @@ +mrd milliárd +millió +milliárd +billió +billiárd +trillió +trilliárd +kvadrillió +kvadrilliárd +kvintillió +kvintilliárd +szextillió +decillió +decilliárd +undecillió +undecilliárd +duodecillió +tridecillió +kvadecillió +kvintdecillió +szexdecillió diff --git a/itn/hungarian/data/number/tens.tsv b/itn/hungarian/data/number/tens.tsv new file mode 100644 index 0000000..2471faa --- /dev/null +++ b/itn/hungarian/data/number/tens.tsv @@ -0,0 +1,9 @@ +tíz 10 +húsz 20 +harminc 30 +negyven 40 +ötven 50 +hatvan 60 +hetven 70 +nyolcvan 80 +kilencven 90 diff --git a/itn/hungarian/data/number/tens_inline.tsv b/itn/hungarian/data/number/tens_inline.tsv new file mode 100644 index 0000000..6f3a07a --- /dev/null +++ b/itn/hungarian/data/number/tens_inline.tsv @@ -0,0 +1,9 @@ +tizen 1 +huszon 2 +harminc 3 +negyven 4 +ötven 5 +hatvan 6 +hetven 7 +nyolcvan 8 +kilencven 9 diff --git a/itn/hungarian/data/number/zero.tsv b/itn/hungarian/data/number/zero.tsv new file mode 100644 index 0000000..a964ff4 --- /dev/null +++ b/itn/hungarian/data/number/zero.tsv @@ -0,0 +1 @@ +nulla 0 diff --git a/itn/hungarian/data/ordinals/endings.tsv b/itn/hungarian/data/ordinals/endings.tsv new file mode 100644 index 0000000..cda9bab --- /dev/null +++ b/itn/hungarian/data/ordinals/endings.tsv @@ -0,0 +1,23 @@ +kilenc kilencedik +nyolc nyolcadik +hét hetedik +hat hatodik +öt ötödik +négy negyedik +három harmadik +kettő kettedik +két kettedik +tíz tizedik +egy egyedik +húsz huszadik +harminc harmincadik +ven venedik +van vanadik +száz századik +ezer ezredik +millió milliomodik +milliárd milliárdodik +billió billiomodik +billiárd billiárdodik +trillió trilliomodik +trilliárd trilliárdodik diff --git a/itn/hungarian/data/ordinals/exceptional.tsv b/itn/hungarian/data/ordinals/exceptional.tsv new file mode 100644 index 0000000..c9cedc0 --- /dev/null +++ b/itn/hungarian/data/ordinals/exceptional.tsv @@ -0,0 +1,4 @@ +egy első +kettő második +két második +nulla nulladik diff --git a/itn/hungarian/data/telephone/area_codes.tsv b/itn/hungarian/data/telephone/area_codes.tsv new file mode 100644 index 0000000..040dc0c --- /dev/null +++ b/itn/hungarian/data/telephone/area_codes.tsv @@ -0,0 +1,71 @@ +1 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +40 +41 +42 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +59 +60 +61 +62 +63 +66 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +99 diff --git a/itn/hungarian/data/telephone/country_codes.tsv b/itn/hungarian/data/telephone/country_codes.tsv new file mode 100644 index 0000000..35aba1b --- /dev/null +++ b/itn/hungarian/data/telephone/country_codes.tsv @@ -0,0 +1,213 @@ +1 +20 +211 +212 +213 +216 +218 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +27 +290 +291 +297 +298 +299 +30 +31 +32 +33 +34 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +36 +370 +371 +372 +373 +374 +375 +376 +377 +378 +380 +381 +382 +383 +385 +386 +387 +389 +39 +40 +41 +420 +421 +423 +43 +44 +45 +46 +47 +48 +49 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +51 +52 +53 +54 +55 +56 +57 +58 +590 +591 +592 +593 +594 +595 +596 +597 +598 +60 +61 +62 +63 +64 +65 +66 +670 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +685 +686 +687 +688 +689 +690 +691 +692 +7 +800 +808 +81 +82 +84 +850 +852 +853 +855 +856 +86 +870 +878 +880 +881 +882 +886 +90 +91 +92 +93 +94 +95 +960 +961 +962 +963 +964 +965 +966 +967 +968 +970 +971 +972 +973 +974 +975 +976 +977 +979 +98 +992 +993 +994 +995 +996 +997 +998 diff --git a/itn/hungarian/data/time/time_zone.tsv b/itn/hungarian/data/time/time_zone.tsv new file mode 100644 index 0000000..b53395b --- /dev/null +++ b/itn/hungarian/data/time/time_zone.tsv @@ -0,0 +1,9 @@ +cst c s t +cet c e t +cest c e s t +pst p s t +est e s t +pt p t +et e t +gmt g m t +bst b s t diff --git a/itn/hungarian/data/whitelist.tsv b/itn/hungarian/data/whitelist.tsv new file mode 100644 index 0000000..c0b2975 --- /dev/null +++ b/itn/hungarian/data/whitelist.tsv @@ -0,0 +1,24 @@ +stb. s a többi +vs. versus +vö. vesd össze +vmi valami +vki valaki +ún. úgynevezett +uő ugyanő +uo. ugyanott +ui. ugyanis +ua. ugyanaz +ti. tudniillik +szül. született +megh. meghalt +sztem szerintem +Szt. Szent +pu. pályaudvar +pl. például +ált. általános +és tsai. és társai +ford. fordította +i. m. idézett mű +ifj. ifjabb +kb. körülbelül +ld. lásd diff --git a/itn/hungarian/data/whitelist_inflect.tsv b/itn/hungarian/data/whitelist_inflect.tsv new file mode 100644 index 0000000..bdae29a --- /dev/null +++ b/itn/hungarian/data/whitelist_inflect.tsv @@ -0,0 +1,39 @@ +m-es méteres +Hz hertz +MHz megahertz +mp másodperc +Mrd milliárd +op. opus +Ft-os forintos +gimn. gimnázium +hsz. határozószó +isk. iskola +ism. ismétlés +ker. kerület +kg-os kilogrammos +kir. király +km-es kilométeres +korm. kormány +köv. következő +Kr. Krisztus +krt. körút +ktg. költség +max. maximum +dkg dekagramm +db darab +em. emelet +évf. évfolyam +fej. fejezet +fn. főnév +fszt. földszint +Ft forint +cm-es centiméteres +cs. csoport +dr. doktor +prof. professzor +u. utca +tv. törvény +tel. telefon +Pf. postafiók +alez. alezredes +bek. bekezdés diff --git a/itn/hungarian/data/whitelist_inflect_sg.tsv b/itn/hungarian/data/whitelist_inflect_sg.tsv new file mode 100644 index 0000000..e2d689a --- /dev/null +++ b/itn/hungarian/data/whitelist_inflect_sg.tsv @@ -0,0 +1,14 @@ +Bp. Budapest +Magyaro. Magyarország +D-DK dél-délkelet +DK délkelet +D-DNy dél-délnyugat +DNy délnyugat +É-ÉK észak-északkelet +ÉK északkelet +É-ÉNy észak-északnyugat +ÉNy északnyugat +K-DK kelet-délkelet +K-ÉK kelet-északkelet +Ny-DNy nyugat-délnyugat +Ny-ÉNy nyugat-északnyugat \ No newline at end of file diff --git a/itn/hungarian/inverse_normalizer.py b/itn/hungarian/inverse_normalizer.py new file mode 100644 index 0000000..c97ceb8 --- /dev/null +++ b/itn/hungarian/inverse_normalizer.py @@ -0,0 +1,93 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from itn.hungarian.rules.cardinal import Cardinal +from itn.hungarian.rules.date import Date +from itn.hungarian.rules.decimal import Decimal +from itn.hungarian.rules.electronic import Electronic +from itn.hungarian.rules.fraction import Fraction +from itn.hungarian.rules.measure import Measure +from itn.hungarian.rules.money import Money +from itn.hungarian.rules.ordinal import Ordinal +from itn.hungarian.rules.punctuation import Punctuation +from itn.hungarian.rules.telephone import Telephone +from itn.hungarian.rules.time import Time +from itn.hungarian.rules.whitelist import WhiteList +from itn.hungarian.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Hungarian rule emits its fields in written order. +TOKEN_ORDERS = {} + +# Written without a space before (closing) or after (opening) the mark. +CLOSING_MARKS = ".,!?:;)]}…»”" +OPENING_MARKS = "([{«„“" + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="hu_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS) + self.build_fst("hu_itn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + ordinal = Ordinal(cardinal) + decimal = Decimal(cardinal) + fraction = Fraction(cardinal, ordinal) + punctuation = Punctuation() + word = Word(punctuation) + rules = ( + RuleSpec(WhiteList(), 1.01), + RuleSpec(Time(cardinal), 1.05), + RuleSpec(Money(cardinal, decimal), 1.07), + RuleSpec(Measure(cardinal, decimal, fraction), 1.08), + RuleSpec(decimal, 1.08), + RuleSpec(fraction, 1.09), + RuleSpec(Date(cardinal), 1.09), + RuleSpec(ordinal, 1.1), + RuleSpec(cardinal, 1.1), + RuleSpec(Telephone(cardinal), 1.1), + RuleSpec(Electronic(), 1.1), + RuleSpec(word, 200), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + space = self.DELETE_EXTRA_SPACE + punct = pynutil.add_weight(punctuation.tagger, 2.1) + punct = (punct + (space.ques + punct).star).optimize() + separator = space | space.ques + punct + space.ques + tokens = (punct + space.ques).ques + classify + (separator + classify).star + (space.ques + punct).ques + self.tagger = self.DELETE_SPACE + (tokens | punct) + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + + # Closing marks attach to the previous token, opening marks to the next. + def marks(chars, graph=None): + chars = pynini.union(*[pynini.escape(mark) for mark in chars]).plus + value = chars if graph is None else pynini.difference(graph, chars) + return punctuation.delete_tokens(punctuation.verbalize_field("value", value)) + + closing = marks(CLOSING_MARKS) + opening = marks(OPENING_MARKS) + other = marks(CLOSING_MARKS, pynini.difference(punctuation.graph, pynini.union( + *[pynini.escape(mark) for mark in OPENING_MARKS]).plus)) + plain = (self.verbalizer_union(rules[:-1]) | other).optimize() + item = opening.star + plain | opening.plus + closing + self.verbalizer = ( + (item | closing) + (self.INSERT_SPACE + item | closing).star + (self.INSERT_SPACE + opening.plus).ques + | opening.plus + ) diff --git a/itn/hungarian/rules/__init__.py b/itn/hungarian/rules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/itn/hungarian/rules/cardinal.py b/itn/hungarian/rules/cardinal.py new file mode 100644 index 0000000..36684c7 --- /dev/null +++ b/itn/hungarian/rules/cardinal.py @@ -0,0 +1,99 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.hungarian.rules.utils import DIGIT, NONZERO, data_map +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Long-scale powers above a thousand, lowest first. +POWERS = ["millió", "milliárd", "billió", "billiárd", "trillió", "trilliárd"] + + +def group_digits(min_grouped=5): + """Writes numbers of at least ``min_grouped`` digits in space-separated triples.""" + + triple = insert(" ") + DIGIT ** 3 + grouped = pynini.closure(DIGIT, 1, 3) + triple.plus + short = pynini.closure(DIGIT, 1, min_grouped - 1) + long = pynini.closure(DIGIT, min_grouped) + return (short | pynini.project(long, "input") @ grouped).optimize() + + +class Cardinal(Processor): + """Cardinals, e.g. ``kettőezer-tíz`` / ``kétezer tíz`` -> ``2010``. + + The tagger keeps the spoken words; the verbalizer writes digits, grouping + numbers of five or more digits with spaces (``tízezer`` -> ``10 000``). + A bare ``egy`` (also the indefinite article) is not tagged on its own. + """ + + def __init__(self): + super().__init__("cardinal", ordertype="itn", token_orders=TOKEN_ORDER) + self.build_numbers() + self.sign = pynini.cross("mínusz", "-") + self.sign_tagger = (self.tag_field("negative", self.sign) + pynini.accep(" ")).ques + self.sign_verbalizer = (self.verbalize_field("negative", self.sign) + delete(" ")).ques + standalone = pynini.difference(pynini.project(self.graph, "input"), "egy") @ self.graph + self.tagger = self.add_tokens(self.sign_tagger + self.tag_field("integer", standalone)) + self.verbalizer = self.delete_tokens(self.sign_verbalizer + self.verbalize_field("integer", self.graph)) + + def build_numbers(self): + """Builds ``self.spoken``: digits -> every accepted spoken form.""" + + two = pynini.cross("2", "két") + digit = data_map("number/digit.tsv", invert=True) + digit_inline = data_map("number/digit_inline.tsv", invert=True) | pynini.cross("2", "kettő") + tens = data_map("number/tens.tsv", invert=True) + tens_inline = data_map("number/tens_inline.tsv", invert=True) + zero = data_map("number/zero.tsv", invert=True) + digit_word = (digit | two).optimize() + self.digit = digit_word + digit_no_one = (NONZERO - "1") @ digit_word + digit_inline_no_one = (NONZERO - "1") @ digit_inline + + graph_tens = (tens_inline + digit_word | tens).optimize() + self.tens = graph_tens + one = pynini.union("", "egy") + base_hundreds = pynini.cross("1", one + "száz") | digit_inline_no_one + insert("száz") + hundreds = (base_hundreds + (delete("00") | graph_tens | delete("0") + digit_word)).optimize() + self.hundreds = hundreds + comp = (hundreds | delete("0") + graph_tens | delete("00") + digit_word).optimize() + comp_no_one = (hundreds | delete("0") + graph_tens | delete("00") + digit_no_one).optimize() + head = (digit_word | graph_tens | hundreds).optimize() + head_no_one = (digit_no_one | graph_tens | hundreds).optimize() + self.up_to_hundreds = head + + joint = insert(pynini.union("-", " ", "")) + level = ( + delete("000") + comp + | comp_no_one + insert("ezer") + (joint + comp | delete("000")) + | pynini.cross("001", one + "ezer") + (joint + comp | delete("000")) + ).optimize() + top = head | (head_no_one + insert("ezer") | pynini.cross("1", one + "ezer")) + (joint + comp | delete("000")) + width = 6 + for word in POWERS: + rest = joint + level | delete("0" * width) + top |= (pynini.cross("1", one + word) | head_no_one + insert(word)) + rest + level = ( + delete("000") + level | (pynini.cross("001", one + word) | comp_no_one + insert(word)) + rest + ).optimize() + width += 3 + self.zero = zero + self.spoken = (zero | top).optimize() + self.digits = pynini.invert(self.spoken).optimize() + self.graph = (self.digits @ group_digits()).optimize() diff --git a/itn/hungarian/rules/date.py b/itn/hungarian/rules/date.py new file mode 100644 index 0000000..5c07556 --- /dev/null +++ b/itn/hungarian/rules/date.py @@ -0,0 +1,66 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import insert + +from itn.hungarian.rules.utils import DIGIT, NONZERO, TO_LOWER, SIGMA, data_labels +from tn.processor import Processor + +TOKEN_ORDER = {} + +CASE_ENDINGS = { + "e": "ét ének ével éért évé éig eként éül ében én énél ébe ére éhez éből éről étől", + "a": "át ának ával áért ává áig aként ául ában án ánál ába ára ához ából áról ától", +} + + +def day_forms(): + """Spoken day -> written day, e.g. ``elseje`` -> ``1.``, ``elsején`` -> ``1-jén``.""" + + written = {} + for number, day in data_labels("dates/days.tsv"): + written.setdefault(day, number + ".") + base = day[:-1] + for ending in CASE_ENDINGS[day[-1]].split(" "): + prefix = "j" if day.endswith("eje") else "" + written.setdefault(base + ending, "{}-{}{}".format(number, prefix, ending)) + written.setdefault(base + "i", "{}-{}i".format(number, "j" if day == "elseje" else "")) + return pynini.string_map([(spoken, number) for spoken, number in written.items()]).optimize() + + +class Date(Processor): + """Dates, e.g. ``kettőezer-tíz április elseje`` -> ``2010. április 1.``. + + Raw spoken ``year``/``month``/``day`` fields, written in Hungarian + year-month-day order with the year dot. + """ + + def __init__(self, cardinal): + super().__init__("date", ordertype="itn", token_orders=TOKEN_ORDER) + names = pynini.union(*[row[1] for row in data_labels("dates/months.tsv")]) + self.month = (names | (TO_LOWER + SIGMA) @ names).optimize() + self.day = day_forms() + self.year = (pynini.invert(cardinal.spoken) @ (NONZERO + DIGIT ** 3) + insert(".")).optimize() + + space = pynini.accep(" ") + year_t = self.tag_field("year", self.year) + space + month_t = self.tag_field("month", self.month) + day_t = space + self.tag_field("day", self.day) + self.tagger = self.add_tokens(year_t.ques + month_t + day_t | year_t + month_t) + + year_v = self.verbalize_field("year", self.year) + space + month_v = self.verbalize_field("month", self.month) + day_v = space + self.verbalize_field("day", self.day) + self.verbalizer = self.delete_tokens(year_v.ques + month_v + day_v.ques) diff --git a/itn/hungarian/rules/decimal.py b/itn/hungarian/rules/decimal.py new file mode 100644 index 0000000..9891a66 --- /dev/null +++ b/itn/hungarian/rules/decimal.py @@ -0,0 +1,80 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.hungarian.rules.utils import DIGIT, NONZERO, data_labels, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Names of the fractional positions after "tized" and "század". +DECIMAL_NAMES = ["ezred", "milliomod", "milliárdod", "billiomod", "billiárdod", "trilliomod", "trilliárdod"] + + +def quantities(): + """Spoken quantity words (with case endings) -> written, e.g. ``milliárd`` -> ``mrd``.""" + + written = {} + for row in data_labels("number/quantities.tsv"): + if len(row) == 2: + pairs = [(row[0], row[1])] + naive_inflector(row[0], row[1], True) + else: + pairs = [(row[0], row[0])] + [(form, form) for _, form in naive_inflector(".", row[0], True)] + for key, spoken in pairs: + written.setdefault(spoken, key) + return pynini.string_map([(spoken, key) for spoken, key in written.items()]).optimize() + + +class Decimal(Processor): + """Decimals, e.g. ``tizenegy egész négyezer-hat százezred`` -> ``11,04006``. + + Raw spoken ``integer_part``/``fractional_part`` and an optional + ``quantity`` (``egy milliárd`` -> ``1 mrd``). + """ + + def __init__(self, cardinal): + super().__init__("decimal", ordertype="itn", token_orders=TOKEN_ORDER) + names = ["tized", "század"] + [prefix + name for name in DECIMAL_NAMES for prefix in ["", "tíz", "száz"]] + spoken = None + leading = pynini.accep("") + for name in names: + # The position of the last digit fixes the denominator. + branch = (leading + NONZERO) @ (delete("0").star + cardinal.spoken) + insert(" " + name) + spoken = branch if spoken is None else spoken | branch + leading += DIGIT + self.fractional = pynini.invert(spoken).optimize() + self.integer = cardinal.graph + self.quantity = quantities() + self.quantity_integer = pynini.invert(cardinal.up_to_hundreds).optimize() + + integer_t = self.tag_field("integer_part", self.integer) + fractional_t = self.tag_field("fractional_part", self.fractional) + quantity_t = pynini.accep(" ") + self.tag_field("quantity", self.quantity) + decimal_t = integer_t + delete(" egész") + pynini.accep(" ") + fractional_t + self.unsigned_tagger = ( + decimal_t + quantity_t.ques + | self.tag_field("integer_part", self.quantity_integer) + quantity_t + ).optimize() + self.tagger = self.add_tokens(cardinal.sign_tagger + self.unsigned_tagger) + + integer_v = self.verbalize_field("integer_part", self.integer) + fractional_v = self.verbalize_field("fractional_part", self.fractional) + quantity_v = pynini.accep(" ") + self.verbalize_field("quantity", self.quantity) + self.unsigned_verbalizer = ( + integer_v + pynini.cross(" ", ",") + fractional_v + quantity_v.ques + | integer_v + quantity_v + ).optimize() + self.verbalizer = self.delete_tokens(cardinal.sign_verbalizer + self.unsigned_verbalizer) diff --git a/itn/hungarian/rules/electronic.py b/itn/hungarian/rules/electronic.py new file mode 100644 index 0000000..f0617e4 --- /dev/null +++ b/itn/hungarian/rules/electronic.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import byte +from pynini.lib.pynutil import delete + +from itn.hungarian.rules.utils import SIGMA, data_keys, data_labels, data_map +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Electronic(Processor): + """Spelled e-mails, URLs and handles, e.g. ``a b c kukac d e f pont hu`` -> ``abc@def.hu``. + + Raw spoken ``protocol``/``username``/``domain`` fields; the verbalizer + joins letters and reads digit and symbol names (``pont`` -> ``.``). + """ + + def __init__(self): + super().__init__("electronic", ordertype="itn", token_orders=TOKEN_ORDER) + symbols = data_map("electronic/symbols.tsv", invert=True) + digits = data_map("number/digit.tsv") | data_map("number/zero.tsv") + letter = byte.ALPHA + char = (letter | digits | symbols).optimize() + servers = data_keys("electronic/server_name.tsv") + tlds = pynini.union(*[row[0][1:] for row in data_labels("electronic/domain.tsv")]) + name = (char | servers | tlds).optimize() + dot = pynini.cross("pont", ".") + joint = delete(" ") + + def spell(piece, minimum=1): + return (piece + pynini.closure(joint + piece, minimum - 1)).optimize() + + piece = pynini.difference(pynini.project(name, "input"), "pont") @ name + domain = spell(piece) + (joint + dot + joint + spell(piece)).plus + self.domain = domain.optimize() + self.username = spell(char) + self.handle = spell(char, 2) + protocol_start = spell(letter) @ pynini.union("http", "https") + joint + pynini.cross("kettőspont", ":") + protocol_start += (joint + pynini.cross("perjel", "/")) ** 2 + www = pynini.cross("w w w", "www") + joint + dot + self.protocol = (protocol_start | www | protocol_start + joint + www).optimize() + # A bare domain must end in a known top-level domain. + site = (SIGMA + " pont " + tlds) @ self.domain + + space = pynini.accep(" ") + domain_t = self.tag_field("domain", self.domain) + email_t = self.tag_field("username", self.username) + delete(" kukac") + space + domain_t + handle_t = delete("kukac ") + self.tag_field("username", self.handle) + url_t = self.tag_field("protocol", self.protocol) + space + domain_t + self.tagger = self.add_tokens(self.tag_field("domain", site) | email_t | handle_t | url_t) + + domain_v = self.verbalize_field("domain", self.domain) + username_v = self.verbalize_field("username", self.username) + email_v = username_v + pynini.cross(" ", "@") + domain_v + handle_v = pynini.cross("", "@") + username_v + url_v = self.verbalize_field("protocol", self.protocol) + delete(" ") + domain_v + self.verbalizer = self.delete_tokens(domain_v | email_v | handle_v | url_v) diff --git a/itn/hungarian/rules/fraction.py b/itn/hungarian/rules/fraction.py new file mode 100644 index 0000000..121c89d --- /dev/null +++ b/itn/hungarian/rules/fraction.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete + +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Fraction(Processor): + """Fractions, e.g. ``kettő harmad`` -> ``2/3``, ``egy és egy fél`` -> ``1 1/2``.""" + + def __init__(self, cardinal, ordinal): + super().__init__("fraction", ordertype="itn", token_orders=TOKEN_ORDER) + numerator = cardinal.graph + denominator = ordinal.fractional + + integer_t = self.tag_field("integer_part", cardinal.graph) + delete(" és") + pynini.accep(" ") + fraction_t = (self.tag_field("numerator", numerator) + pynini.accep(" ") + + self.tag_field("denominator", denominator)) + self.unsigned_tagger = (integer_t.ques + fraction_t).optimize() + self.tagger = self.add_tokens(cardinal.sign_tagger + self.unsigned_tagger) + + integer_v = self.verbalize_field("integer_part", cardinal.graph) + pynini.accep(" ") + fraction_v = (self.verbalize_field("numerator", numerator) + pynini.cross(" ", "/") + + self.verbalize_field("denominator", denominator)) + self.unsigned_verbalizer = (integer_v.ques + fraction_v).optimize() + self.verbalizer = self.delete_tokens(cardinal.sign_verbalizer + self.unsigned_verbalizer) diff --git a/itn/hungarian/rules/measure.py b/itn/hungarian/rules/measure.py new file mode 100644 index 0000000..30266ec --- /dev/null +++ b/itn/hungarian/rules/measure.py @@ -0,0 +1,74 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete + +from itn.hungarian.rules.utils import data_labels, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Clock units are written by the time rule. +SKIPPED_UNITS = {"ó", "mp"} + + +def unit_rows(): + """Unit rows with lower-case abbreviations (``l`` before ``L``) first, otherwise in file order.""" + + rows = data_labels("measures/measurements.tsv") + return sorted(rows, key=lambda row: row[0] != row[0].lower()) + + +def units(): + """Spoken units (with case endings) -> abbreviations; the first abbreviation wins.""" + + written = {} + for row in unit_rows(): + key, word = row[0], row[1] + if key in SKIPPED_UNITS: + continue + written.setdefault(word, key) + if " " not in word: + for abbr, spoken in naive_inflector(key.lower(), word): + written.setdefault(spoken, abbr) + return pynini.string_map([(spoken, abbr) for spoken, abbr in written.items()]).optimize() + + +class Measure(Processor): + """Measures, e.g. ``három milliméterben`` -> ``3mm-ben``, ``mínusz tizenkettő kilogramm`` -> ``-12kg``. + + The number fields and the spoken (possibly inflected) unit stay raw; the + unit abbreviation is attached to the number. + """ + + def __init__(self, cardinal, decimal, fraction): + super().__init__("measure", ordertype="itn", token_orders=TOKEN_ORDER) + unit = units() + bare = {} + for row in unit_rows(): + if " " not in row[1]: + bare.setdefault(row[1], row[0]) + bare = pynini.string_map(list(bare.items())) + per = pynini.cross(" per ", "/") + bare + self.units = (unit + per.ques).optimize() + + number_t = self.tag_field("integer", cardinal.graph) | decimal.unsigned_tagger | fraction.unsigned_tagger + units_t = pynini.accep(" ") + self.tag_field("units", self.units) + self.tagger = self.add_tokens(cardinal.sign_tagger + number_t + units_t) + + number_v = (self.verbalize_field("integer", cardinal.graph) + | decimal.unsigned_verbalizer | fraction.unsigned_verbalizer) + units_v = delete(" ") + self.verbalize_field("units", self.units) + self.verbalizer = self.delete_tokens(cardinal.sign_verbalizer + number_v + units_v) diff --git a/itn/hungarian/rules/money.py b/itn/hungarian/rules/money.py new file mode 100644 index 0000000..44b875a --- /dev/null +++ b/itn/hungarian/rules/money.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.hungarian.rules.utils import DIGIT, data_labels, inflect_abbreviation, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {} + + +def currencies(): + """Returns ``(spoken name, written symbol, minor unit)`` with the first listed symbol per name.""" + + minors = dict((row[0], row[1]) for row in data_labels("money/currency_minor.tsv")) + seen = {} + for symbol, name in data_labels("money/currency.tsv"): + seen.setdefault(name, (symbol, minors[symbol])) + return [(name, symbol, minor) for name, (symbol, minor) in seen.items()] + + +class Money(Processor): + """Money, e.g. ``egy dollár egy cent`` -> ``$1,01``, ``egy milliárd angol fontot`` -> ``1 mrd GBP-t``. + + Raw spoken amount fields, ``currency`` and optional ``minor`` amount with + its ``minor_currency``. Symbols (``$``, ``£``) are written before the + amount, letter codes (``GBP``, ``Ft``) after it with their case endings. + """ + + def __init__(self, cardinal, decimal): + super().__init__("money", ordertype="itn", token_orders=TOKEN_ORDER) + cents = (pynini.invert(cardinal.spoken) @ pynini.closure(DIGIT, 1, 2) + @ (insert("0") + DIGIT | DIGIT ** 2)).optimize() + integer_t = self.tag_field("integer_part", cardinal.graph) + amount_t = integer_t | decimal.unsigned_tagger + integer_v = self.verbalize_field("integer_part", cardinal.graph) + amount_v = integer_v | decimal.unsigned_verbalizer + + taggers, verbalizers = [], [] + for name, symbol, minor in currencies(): + symbol_written = pynini.escape(symbol) + prefix = re.search(r"[^A-Za-z]", symbol) is not None + forms = [(symbol, name)] + if not prefix: + inflected = inflect_abbreviation(symbol, name) if re.match("^[A-Z]{3}$", symbol) else [] + forms += inflected + naive_inflector(symbol, name) + written = {} + for abbr, spoken in forms: + written.setdefault(spoken, abbr) + currency = pynini.string_map([(spoken, abbr) for spoken, abbr in written.items()]).optimize() + bare = pynini.cross(pynini.escape(name), symbol_written) + minor_t = (pynini.accep(" ") + self.tag_field("minor", cents) + pynini.accep(" ") + + self.tag_field("minor_currency", pynini.accep(minor))) + minor_v = (pynini.cross(" ", ",") + self.verbalize_field("minor", cents) + + self.verbalize_field("minor_currency", delete(minor), leading_space=True)) + taggers.append(amount_t + pynini.accep(" ") + self.tag_field("currency", currency)) + taggers.append(integer_t + pynini.accep(" ") + self.tag_field("currency", bare) + minor_t) + if prefix: + delete_currency = self.verbalize_field("currency", delete(pynini.escape(name)), leading_space=True) + verbalizers.append(insert(symbol_written) + amount_v + delete_currency) + verbalizers.append(insert(symbol_written) + integer_v + delete_currency + minor_v) + else: + currency_v = pynini.accep(" ") + self.verbalize_field("currency", currency) + verbalizers.append(amount_v + currency_v) + bare_v = self.verbalize_field("currency", delete(pynini.escape(name)), leading_space=True) + verbalizers.append(integer_v + bare_v + minor_v + insert(" " + symbol_written)) + self.tagger = self.add_tokens(pynini.union(*taggers)) + self.verbalizer = self.delete_tokens(pynini.union(*verbalizers)) diff --git a/itn/hungarian/rules/ordinal.py b/itn/hungarian/rules/ordinal.py new file mode 100644 index 0000000..0d29f4e --- /dev/null +++ b/itn/hungarian/rules/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.hungarian.rules.utils import data_map, suffix_rewrite +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Ordinal(Processor): + """Ordinals, e.g. ``harmincegyedik`` -> ``31.``. + + The spoken word stays raw; the verbalizer writes plain digits and the + ordinal dot. A bare ``első`` ("first") is not tagged on its own. + """ + + def __init__(self, cardinal): + super().__init__("ordinal", ordertype="itn", token_orders=TOKEN_ORDER) + to_ordinal = suffix_rewrite(data_map("ordinals/endings.tsv"), data_map("ordinals/exceptional.tsv")) + spoken = (cardinal.spoken @ to_ordinal).optimize() + self.graph = (pynini.invert(spoken) + insert(".")).optimize() + # Denominators: "harmad" -> "3", "fél" -> "2". + exceptions = pynini.string_map([("első", "egyed"), ("második", "fél")]) + fractional = spoken @ suffix_rewrite(delete("ik"), exceptions) + self.fractional = pynini.invert(fractional).optimize() + standalone = pynini.difference(pynini.project(self.graph, "input"), "első") @ self.graph + self.tagger = self.add_tokens(self.tag_field("integer", standalone)) + self.verbalizer = self.delete_tokens(self.verbalize_field("integer", self.graph)) diff --git a/itn/hungarian/rules/punctuation.py b/itn/hungarian/rules/punctuation.py new file mode 100644 index 0000000..fe0feeb --- /dev/null +++ b/itn/hungarian/rules/punctuation.py @@ -0,0 +1,44 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini + +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Symbols that other rules read aloud are not split off as punctuation. +EXCLUDED = set("&#@§™®©_%*+/=^|~$£€₩¥°º") + + +class Punctuation(Processor): + """Punctuation marks are kept verbatim.""" + + def __init__(self): + super().__init__("punctuation", ordertype="itn", token_orders=TOKEN_ORDER) + unicode_marks = [chr(i) for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in ("[", "]")] + ascii_marks = list("!#%&'()*+,-./:;<=>?@^_`{|}~\"") + self.punct_marks = [mark for mark in unicode_marks + ["[", "]"] + ascii_marks if mark not in EXCLUDED] + self.punct = pynini.union(*[pynini.escape(mark) for mark in self.punct_marks]).optimize() + whitespace = pynini.union(" ", "\t", "\n", "\r", "\u00a0") + text = (self.VCHAR - whitespace - pynini.union("<", ">")).plus + emphasis = pynini.accep("<") + (text + pynini.accep("/").ques | pynini.accep("/") + text) + ">" + self.graph = (self.punct.plus | emphasis).optimize() + self.tagger = self.add_tokens(self.tag_field("value", self.graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.graph)) diff --git a/itn/hungarian/rules/telephone.py b/itn/hungarian/rules/telephone.py new file mode 100644 index 0000000..567413d --- /dev/null +++ b/itn/hungarian/rules/telephone.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete + +from itn.hungarian.rules.utils import DIGIT, NONZERO, data_keys +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Telephone(Processor): + """Phone numbers and IPv4 addresses, e.g. ``plusz harminchat egy ...`` -> ``+36 1 4888 700``. + + A number starts with a country code (``plusz harminchat``) or the + domestic trunk prefix (``nulla hat``); raw spoken ``country_code`` and + ``number_part`` fields are written as space-separated digit groups. + """ + + def __init__(self, cardinal): + super().__init__("telephone", ordertype="itn", token_orders=TOKEN_ORDER) + digits = pynini.invert(cardinal.spoken) + zero = pynini.cross("nulla", "0") + delete(" ") + digit = digits @ NONZERO + two = (digits @ (NONZERO + DIGIT) | zero + digit).optimize() + tens = digits @ (NONZERO + DIGIT) + three = (digits @ (NONZERO + DIGIT ** 2) | zero + tens | zero + zero + digit).optimize() + four = (digits @ (NONZERO + DIGIT ** 3) | zero + three).optimize() + up_to_three = (digits @ pynini.closure(DIGIT, 1, 3)).optimize() + + space = pynini.accep(" ") + area = digits @ data_keys("telephone/area_codes.tsv") + base = pynini.union( + three + space + three, + three + space + two + space + two, + three + space + four, + two + space + four, + four + space + three, + two + space + two + space + three, + ) + trunk = pynini.cross("nulla hat", "06") + number = area + space + base + ip = up_to_three + (pynini.cross(" pont ", ".") + up_to_three) ** 3 + self.number = (number | ip).optimize() + self.trunk_number = (trunk + space + number).optimize() + country = digits @ data_keys("telephone/country_codes.tsv") + self.country = (pynini.cross("plusz ", "+") + country).optimize() + + country_t = self.tag_field("country_code", self.country) + space + number_t = self.tag_field("number_part", self.number) + trunk_t = self.tag_field("number_part", self.trunk_number) + self.tagger = self.add_tokens(country_t + number_t | trunk_t | self.tag_field("number_part", ip)) + + country_v = self.verbalize_field("country_code", self.country) + space + number_v = self.verbalize_field("number_part", self.number | self.trunk_number) + self.verbalizer = self.delete_tokens(country_v.ques + number_v) diff --git a/itn/hungarian/rules/time.py b/itn/hungarian/rules/time.py new file mode 100644 index 0000000..1ce698f --- /dev/null +++ b/itn/hungarian/rules/time.py @@ -0,0 +1,99 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.hungarian.rules.utils import DIGIT, data_labels, inflect_abbreviation, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {} + + +def inflected_units(abbr, word): + """Inflected spoken unit -> abbreviation, e.g. ``perccel`` -> ``p-cel``.""" + + written = {} + for form, spoken in naive_inflector(abbr, word, True): + written.setdefault(spoken, form) + return pynini.string_map([(spoken, form) for spoken, form in written.items()]).optimize() + + +class Time(Processor): + """Times, e.g. ``nyolc óra öt perc`` -> ``08:05``, ``nyolc órában`` -> ``8 ó-ban``. + + Raw spoken ``hours``/``minutes``/``seconds``; an inflected trailing unit + stays in ``suffix`` and a spelled time zone in ``zone``. + """ + + def __init__(self, cardinal): + super().__init__("time", ordertype="itn", token_orders=TOKEN_ORDER) + digits = pynini.invert(cardinal.spoken) + + def numbers(first, last): + return digits @ pynini.union(*[str(x) for x in range(first, last + 1)]) + + pad = insert("0") + DIGIT | DIGIT ** 2 + self.hour = numbers(0, 24).optimize() + self.minute = numbers(1, 59).optimize() + hour_padded = (self.hour @ pad).optimize() + minute_padded = (self.minute @ pad).optimize() + + zone = {} + for key, spelled in data_labels("time/time_zone.tsv"): + zone.setdefault(spelled, key) + for form, _ in inflect_abbreviation(spelled, "a", True): + zone.setdefault(form, form.replace(" ", "")) + self.zone = pynini.string_map(list(zone.items())).optimize() + hour_suffix = inflected_units("ó", "óra") + minute_suffix = inflected_units("p", "perc") + second_suffix = inflected_units("mp", "másodperc") + self.suffix = (hour_suffix | minute_suffix | second_suffix).optimize() + + space = pynini.accep(" ") + hours_t = self.tag_field("hours", self.hour) + minutes_t = space + self.tag_field("minutes", self.minute) + seconds_t = space + self.tag_field("seconds", self.minute) + zone_t = (space + self.tag_field("zone", self.zone)).ques + hour_word = delete(" óra") + minute_word = delete(" perc") + second_word = delete(pynini.union(" másodperc", " masodperc")) + + def suffix_t(graph): + return space + self.tag_field("suffix", graph) + + clock = hours_t + hour_word + (minutes_t + minute_word).ques + (seconds_t + second_word).ques + zone_t + spoken = hours_t + suffix_t(hour_suffix) + spoken |= hours_t + hour_word + minutes_t + suffix_t(minute_suffix) + spoken |= hours_t + hour_word + minutes_t + minute_word + seconds_t + suffix_t(second_suffix) + self.tagger = self.add_tokens(clock | spoken) + + hours_v = self.verbalize_field("hours", hour_padded) + zero = insert(":00") + + def clock_v(name, graph): + return pynini.cross(" ", ":") + self.verbalize_field(name, graph) + + minutes_v = clock_v("minutes", minute_padded) + seconds_v = clock_v("seconds", minute_padded) + clock_graph = hours_v + (minutes_v | zero) | hours_v + (minutes_v | zero) + seconds_v + zone_v = (space + self.verbalize_field("zone", self.zone)).ques + suffix_v = space + self.verbalize_field("suffix", self.suffix) + bare_hours = self.verbalize_field("hours", self.hour) + bare = space + self.verbalize_field("minutes", self.minute) + bare_seconds = space + self.verbalize_field("seconds", self.minute) + spoken_v = bare_hours + suffix_v + spoken_v |= bare_hours + insert(" ó") + bare + suffix_v + spoken_v |= bare_hours + insert(" ó") + bare + insert(" p") + bare_seconds + suffix_v + self.verbalizer = self.delete_tokens(clock_graph + zone_v | spoken_v) diff --git a/itn/hungarian/rules/utils.py b/itn/hungarian/rules/utils.py new file mode 100644 index 0000000..6b026ea --- /dev/null +++ b/itn/hungarian/rules/utils.py @@ -0,0 +1,143 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hungarian alphabet, data loading and the naive suffix inflector.""" + +from functools import lru_cache + +import pynini +from pynini.lib import byte, utf8 + +from itn.utils import get_abs_path +from tn.utils import load_labels + +SIGMA = utf8.VALID_UTF8_CHAR.star +DIGIT = byte.DIGIT +NONZERO = pynini.difference(DIGIT, "0").optimize() +_ALPHA_UPPER = "AÁBCDEÉFGHIÍJKLMNOÓÖŐPQRSTUÚÜŰVWXYZ" +_ALPHA_LOWER = "aábcdeéfghiíjklmnoóöőpqrstuúüűvwxyz" +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(_ALPHA_UPPER, _ALPHA_LOWER)]).optimize() + + +def data_path(name): + return get_abs_path("hungarian/data/" + name) + + +def data_labels(name): + return [row for row in load_labels(data_path(name)) if row] + + +def data_map(name, invert=False): + """Maps column 0 to column 1 (or the reverse); one-column rows map to themselves.""" + + pairs = [] + for row in data_labels(name): + key, value = row[0], row[1] if len(row) > 1 else row[0] + pairs.append((value, key) if invert else (key, value)) + return pynini.string_map(pairs).optimize() + + +def data_keys(name): + return pynini.union(*[pynini.escape(row[0]) for row in data_labels(name)]).optimize() + + +def suffix_rewrite(mapping, exceptions=None): + """Rewrites the end of a bounded word with ``mapping``. + + ``exceptions`` (whole-word replacements) take precedence; the result is + meant to be composed onto a bounded spoken graph. + """ + + graph = SIGMA + mapping + if exceptions is None: + return graph.optimize() + keys = pynini.project(exceptions, "input") + return (exceptions | pynini.difference(SIGMA, keys) @ graph).optimize() + + +@lru_cache(maxsize=None) +def _inflection_tables(): + def load(name): + return {row[0]: row[1].split(" ") for row in data_labels("inflection/" + name)} + + return load("endings.tsv"), load("plural_endings.tsv"), load("word_endings.tsv") + + +def _modify_ending(outword, word, form): + """Adjusts endings that are written differently after abbreviations.""" + + if outword == word: + return form + undouble = {"nny": "ny", "lly": "ly"} + for ending in ["ny", "nny", "ly", "lly", "ev", "év", "út", "ut", "a", "á", "e", "é"]: + if form.startswith(ending): + return undouble.get(ending, "") + form[len(ending):] + return form + + +def naive_inflector(abbr, word, singular_only=False): + """Inflects an abbreviation together with its expansion. + + Returns ``(inflected abbreviation, inflected expansion)`` pairs, e.g. + ``("tv.-t", "törvényt")``; possessive forms are omitted. + """ + + singular, plural, lexical = _inflection_tables() + keys_sorted = sorted(singular, key=len, reverse=True) + if word in lexical: + key, ends = word, lexical[word] + else: + key = next((key for key in keys_sorted if word.endswith(key)), None) + if key is None: + raise KeyError("no inflection class for {}".format(word)) + ends = singular[key] + outword = word + for wordend in ["ny", "ly", "év", "út", "a", "e"]: + if outword.endswith(wordend): + outword = outword[:-len(wordend)] + if "-" in abbr: + abbr = abbr.split("-")[0] + forms = [("{}-{}".format(abbr, _modify_ending(outword, word, form)), outword + form) for form in ends] + if not singular_only: + for plural_form in plural[key]: + plural_key = key + "k" if plural_form == "k" else plural_form + tweaked = _modify_ending(outword, word, plural_form) + forms.append(("{}-{}".format(abbr, tweaked), outword + plural_form)) + for form in singular[plural_key]: + forms.append(("{}-{}{}".format(abbr, tweaked, form), outword + plural_form + form)) + return forms + + +def inflect_abbreviation(abbr, word, singular_only=False): + """Inflects an abbreviation by the name of its last letter. + + Returns ``(inflected abbreviation, inflected expansion)`` pairs, e.g. + ``("GBP-t", "angol fontot")``. + """ + + last = abbr[-1].lower() + if last in "bcdgjptvz": + ending = "é" + elif last in "aáeéiíoóöőuúüű": + ending = last + elif last in "flmnrs": + ending = "e" + last + elif last in "hk": + ending = "á" + else: + return [] + word_part = [form[1] for form in naive_inflector(".", word, singular_only)] + abbr_part = [form[0] for form in naive_inflector(abbr, ending, singular_only)] + return list(zip(abbr_part, word_part)) diff --git a/itn/hungarian/rules/whitelist.py b/itn/hungarian/rules/whitelist.py new file mode 100644 index 0000000..470d221 --- /dev/null +++ b/itn/hungarian/rules/whitelist.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from itn.hungarian.rules.utils import data_labels, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Abbreviations whose expansions other rules write (quantities, units). +SKIPPED = {"Mrd", "mp"} + + +def abbreviations(): + """Spoken expansion (with case endings) -> abbreviation; the first listed abbreviation wins.""" + + written = {} + tables = [("whitelist.tsv", None), ("whitelist_inflect.tsv", False), ("whitelist_inflect_sg.tsv", True)] + for name, singular_only in tables: + for row in data_labels(name): + abbr, expansion = row[0], row[1] + if abbr in SKIPPED: + continue + written.setdefault(expansion, abbr) + if singular_only is not None and " " not in expansion: + for form, spoken in naive_inflector(abbr, expansion, singular_only): + written.setdefault(spoken, form) + return pynini.string_map(list(written.items())).optimize() + + +class WhiteList(Processor): + """Abbreviations with their case endings, e.g. ``törvénnyel`` -> ``tv.-nyel``.""" + + def __init__(self): + super().__init__("whitelist", ordertype="itn", token_orders=TOKEN_ORDER) + self.graph = abbreviations() + self.tagger = self.add_tokens(self.tag_field("value", self.graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.graph)) diff --git a/itn/hungarian/rules/word.py b/itn/hungarian/rules/word.py new file mode 100644 index 0000000..aab3232 --- /dev/null +++ b/itn/hungarian/rules/word.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Word(Processor): + """Any other non-space run is copied verbatim.""" + + def __init__(self, punctuation): + super().__init__("word", ordertype="itn", token_orders=TOKEN_ORDER) + whitespace = pynini.union(" ", "\t", "\n", "\r", "\u00a0") + # Exclude whole punctuation tokens, not punctuation embedded in words. + self.graph = ((self.VCHAR - whitespace).plus - punctuation.graph).optimize() + self.value_graph = ( + ((self.VCHAR - " ").plus - self.SPACE.plus) + @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", self.VSIGMA) + | pynutil.delete(self.SPACE.plus) + ).optimize() + self.tagger = self.add_tokens(self.tag_field("value", self.graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.value_graph)) diff --git a/itn/hungarian/test/__init__.py b/itn/hungarian/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/itn/hungarian/test/normalizer_test.py b/itn/hungarian/test/normalizer_test.py new file mode 100644 index 0000000..e58086f --- /dev/null +++ b/itn/hungarian/test/normalizer_test.py @@ -0,0 +1,214 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from itn.hungarian.inverse_normalizer import InverseNormalizer + +# NeMo has no Hungarian ITN fixtures: the TN fixtures are read in reverse. +TN_DATA = Path(__file__).resolve().parents[3] / "tn" / "hungarian" / "test" / "data" + +# Reversed TN pairs that cannot hold, each with the reason. +EXCLUDED = { + "kettőmillió": "only the dot-grouped '2.000.000' is listed; ITN groups with spaces like '1 034 068'", + "milliárd": "only the dot-grouped '1.000.000.000' is listed; ITN groups with spaces", + "kettőmilliárd": "only the dot-grouped '2.000.000.000' is listed; ITN groups with spaces", + "százharmincegyezer-száztizenegyedik": "only '131 111.' is listed; ITN writes ordinals ungrouped like '455456.'", + "mínusz tizenegy egész négyezer-hat tízezred milliárd": + "only '-11,4006 milliárd' is listed; ITN abbreviates milliárd as in '$1 mrd' and '1 mrd GBP-t'", + "kettő egész három tized milliméter": "only '2,3 mm' is listed; ITN attaches units as in '3mm' and '12kg'", + "kettőezer-tíz április elsején": "only numeric months ('2010. 04. 1-jén') are listed; ITN keeps month names", + "április elsején": "only numeric months ('04. 1-jén') are listed; ITN keeps month names", + "plusz harminchat egy négyszáznegyvenegy négyezer": + "the listed '+ (36) 1 441-4000' layout is not recoverable from speech", + "plusz harminchat nyolcvan ötszázötvenöt száztizenegy": + "'+36-80/555-111' conflicts with the space-separated '+36 1 4888 700' layout", + "telefon plusz harminchat nyolcvan ötszázötvenöt száztizenegy": + "'tel.: +36-80/555-111' uses the dash/slash layout and a colon", + "telefon nulla hat negyven negyvenkilenc négyezer-kilencszáznegyvenkilenc": + "'tel. 06-40-49-4949' uses the dash layout; ITN writes '06 40 49 4949'", +} + + +def reversed_cases(): + candidates = {} + lines = 0 + for path in sorted(TN_DATA.glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for line in stream: + if line.endswith("\n"): + line = line[:-1] + written, spoken = line.split("~") + candidates.setdefault(spoken, []).append(written) + lines += 1 + return lines, candidates + + +LINES, CANDIDATES = reversed_cases() +CASES = [pytest.param(spoken, written, id=spoken or "empty") + for spoken, written in CANDIDATES.items() if spoken not in EXCLUDED] + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(InverseNormalizer) + + +def test_reversed_fixture_count(): + assert LINES == 205 + assert len(CANDIDATES) == 178 + assert set(EXCLUDED) <= set(CANDIDATES) + assert len(CASES) == 166 + assert CANDIDATES["ezer"] == ["1000", "1 000", "1.000"] + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_reversed(normalizer, spoken, written): + assert normalizer.normalize(spoken) in written + + +@pytest.mark.parametrize("spoken", sorted(EXCLUDED)) +def test_excluded_pairs_still_fail(normalizer, spoken): + # Keeps the exclusion list minimal: an exclusion that starts passing must be removed. + assert normalizer.normalize(spoken) not in CANDIDATES[spoken] + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_reversed_mapping(normalizer, spoken, written): + result = normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text in written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for spoken in CANDIDATES: + assert normalizer.verbalize(normalizer.tag(spoken)) == normalizer.normalize(spoken) + + +@pytest.mark.parametrize("spoken, kind, members, written", [ + ("kétezer tíz", "cardinal", {"integer": "kétezer tíz"}, "2010"), + ("tízezer", "cardinal", {"integer": "tízezer"}, "10 000"), + ("mínusz öt", "cardinal", {"negative": "mínusz", "integer": "öt"}, "-5"), + ("huszadik", "ordinal", {"integer": "huszadik"}, "20."), + ("tizenegy egész négyezer-hat százezred", "decimal", + {"integer_part": "tizenegy", "fractional_part": "négyezer-hat százezred"}, "11,04006"), + ("kettő egész öt tized millió", "decimal", + {"integer_part": "kettő", "fractional_part": "öt tized", "quantity": "millió"}, "2,5 millió"), + ("egy milliárd", "decimal", {"integer_part": "egy", "quantity": "milliárd"}, "1 mrd"), + ("három és egy negyed", "fraction", {"integer_part": "három", "numerator": "egy", "denominator": "negyed"}, + "3 1/4"), + ("kettő harmad", "fraction", {"numerator": "kettő", "denominator": "harmad"}, "2/3"), + ("három milliméterben", "measure", {"integer": "három", "units": "milliméterben"}, "3mm-ben"), + ("tíz kilométer per óra", "measure", {"integer": "tíz", "units": "kilométer per óra"}, "10km/ó"), + ("öt százalék", "measure", {"integer": "öt", "units": "százalék"}, "5%"), + ("egy dollár egy cent", "money", + {"integer_part": "egy", "currency": "dollár", "minor": "egy", "minor_currency": "cent"}, "$1,01"), + ("egy milliárd angol fontot", "money", + {"integer_part": "egy", "quantity": "milliárd", "currency": "angol fontot"}, "1 mrd GBP-t"), + ("száz forint ötven fillér", "money", + {"integer_part": "száz", "currency": "forint", "minor": "ötven", "minor_currency": "fillér"}, "100,50 Ft"), + ("ezerkilencszáznegyvennyolc március tizenötödike", "date", + {"year": "ezerkilencszáznegyvennyolc", "month": "március", "day": "tizenötödike"}, "1948. március 15."), + ("május elsején", "date", {"month": "május", "day": "elsején"}, "május 1-jén"), + ("kettőezer-huszonhat október", "date", {"year": "kettőezer-huszonhat", "month": "október"}, "2026. október"), + ("tizenkét óra harminc perc", "time", {"hours": "tizenkét", "minutes": "harminc"}, "12:30"), + ("nyolc óra öt másodperc", "time", {"hours": "nyolc", "seconds": "öt"}, "08:00:05"), + ("nyolc órakor", "time", {"hours": "nyolc", "suffix": "órakor"}, "8 ó-kor"), + ("nulla óra c e t-t", "time", {"hours": "nulla", "zone": "c e t-t"}, "00:00 cet-t"), + ("plusz harminchat húsz ötszázötvenöt hatvanhat hatvanhat", "telephone", + {"country_code": "plusz harminchat", "number_part": "húsz ötszázötvenöt hatvanhat hatvanhat"}, + "+36 20 555 66 66"), + ("nulla hat egy négyszáznegyvenegy négyezer", "telephone", + {"number_part": "nulla hat egy négyszáznegyvenegy négyezer"}, "06 1 441 4000"), + ("x kukac y pont com", "electronic", {"username": "x", "domain": "y pont com"}, "x@y.com"), + ("index pont hu", "electronic", {"domain": "index pont hu"}, "index.hu"), + ("kukac j e n s e n pont m e", "electronic", {"username": "j e n s e n pont m e"}, "@jensen.me"), + ("s a többi", "whitelist", {"value": "s a többi"}, "stb."), + ("törvénnyel", "whitelist", {"value": "törvénnyel"}, "tv.-nyel"), + ("köszönöm", "word", {"value": "köszönöm"}, "köszönöm"), +]) +def test_raw_fields(normalizer, spoken, kind, members, written): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == written + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, written", [ + # A bare "egy" is the indefinite article and a bare "első" means "first". + ("egy alma", "egy alma"), + ("az első", "az első"), + ("két alma", "2 alma"), + ("egyezer", "1000"), + ("egymillió kétezer-egy", "1 002 001"), + ("kukac a", "kukac a"), + ("a b c", "a b c"), + ("öt ( hat ) .", "5 (6)."), + ("öt , hat", "5, 6"), + ("két liter", "2l"), + ("egy egész öt század dollár", "$1,05"), +]) +def test_additional_semantics(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken", ["kettő harmad", "nyolc óra öt perc", "egy dollár egy cent", "tízezer", + "április elseje", "a b c kukac d e f pont hu"]) +def test_nbest_mapping_synchronization(normalizer, spoken): + outputs = normalizer.normalize(spoken, nbest=3) + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(spoken) + for result in mapped: + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + spoken = 'út\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(spoken)) + assert [token.members for token in parser.tokens] == [{"value": spoken}] + assert normalizer.normalize(spoken) == spoken + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, spoken): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": spoken} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(spoken, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) diff --git a/pyproject.toml b/pyproject.toml index 738e8c9..7925d9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,8 @@ tn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "hungarian/data/*.tsv", + "hungarian/data/*/*.tsv", "hebrew/data/*.tsv", "hebrew/data/*/*.tsv", "italian/data/*.tsv", @@ -109,6 +111,8 @@ itn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "hungarian/data/*.tsv", + "hungarian/data/*/*.tsv", "hebrew/data/*.tsv", "hebrew/data/*/*.tsv", "italian/data/*.tsv", diff --git a/tn/chinese/test/cache_test.py b/tn/chinese/test/cache_test.py index cf84ef9..3fa39d2 100644 --- a/tn/chinese/test/cache_test.py +++ b/tn/chinese/test/cache_test.py @@ -12,6 +12,7 @@ from itn.hebrew.inverse_normalizer import InverseNormalizer as HeInverseNormalizer from itn.hindi.inverse_normalizer import InverseNormalizer as HiInverseNormalizer from itn.hindi_english.inverse_normalizer import InverseNormalizer as HiEnInverseNormalizer +from itn.hungarian.inverse_normalizer import InverseNormalizer as HuInverseNormalizer from itn.italian.inverse_normalizer import InverseNormalizer as ItInverseNormalizer from itn.kinyarwanda.inverse_normalizer import InverseNormalizer as RwInverseNormalizer from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer @@ -30,6 +31,7 @@ from tn.german.normalizer import Normalizer as DeNormalizer from tn.hebrew.normalizer import Normalizer as HeNormalizer from tn.hindi.normalizer import Normalizer as HiNormalizer +from tn.hungarian.normalizer import Normalizer as HuNormalizer from tn.italian.normalizer import Normalizer as ItNormalizer from tn.kinyarwanda.normalizer import Normalizer as RwNormalizer from tn.japanese.normalizer import Normalizer as JaNormalizer @@ -105,6 +107,7 @@ def _build_small_graphs(processor): (EsNormalizer, "es_tn", "tn", {}, {"deterministic": True}), (EsNormalizer, "es_tn", "tn", {"deterministic": False}, {"deterministic": False}), (HiNormalizer, "hi_tn", "tn", {}, {}), + (HuNormalizer, "hu_tn", "tn", {}, {}), (HeNormalizer, "he_tn", "tn", {}, {}), (ItNormalizer, "it_tn", "tn", {}, {}), (MrNormalizer, "mr_tn", "tn", {}, {}), @@ -156,6 +159,7 @@ def _build_small_graphs(processor): (EsInverseNormalizer, "es_itn", "itn", {}, {"cased": False}), (EsInverseNormalizer, "es_itn", "itn", {"cased": True}, {"cased": True}), (HiInverseNormalizer, "hi_itn", "itn", {}, {}), + (HuInverseNormalizer, "hu_itn", "itn", {}, {}), (HeInverseNormalizer, "he_itn", "itn", {}, {}), (ItInverseNormalizer, "it_itn", "itn", {}, {}), (HiEnInverseNormalizer, "hi_en_itn", "itn", {}, {"cased": False}), diff --git a/tn/cli.py b/tn/cli.py index a9d3a21..6f69abd 100644 --- a/tn/cli.py +++ b/tn/cli.py @@ -69,6 +69,7 @@ class PipelineSpec: (BooleanOption("deterministic", True, "use deterministic Spanish readings"),), ), "hi": PipelineSpec("tn.hindi.normalizer", "Normalizer"), + "hu": PipelineSpec("tn.hungarian.normalizer", "Normalizer"), "he": PipelineSpec("tn.hebrew.normalizer", "Normalizer"), "it": PipelineSpec("tn.italian.normalizer", "Normalizer"), "mr": PipelineSpec("tn.marathi.normalizer", "Normalizer"), @@ -111,6 +112,7 @@ class PipelineSpec: (BooleanOption("cased", False, 'also accept capitalized spoken words, e.g. "Doce Dólares"'),), ), "hi": PipelineSpec("itn.hindi.inverse_normalizer", "InverseNormalizer"), + "hu": PipelineSpec("itn.hungarian.inverse_normalizer", "InverseNormalizer"), "he": PipelineSpec("itn.hebrew.inverse_normalizer", "InverseNormalizer"), "it": PipelineSpec("itn.italian.inverse_normalizer", "InverseNormalizer"), "hi_en": PipelineSpec( diff --git a/tn/hungarian/__init__.py b/tn/hungarian/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tn/hungarian/data/dates/days.tsv b/tn/hungarian/data/dates/days.tsv new file mode 100644 index 0000000..f402f03 --- /dev/null +++ b/tn/hungarian/data/dates/days.tsv @@ -0,0 +1,31 @@ +1 elseje +2 másodika +3 harmadika +4 negyedike +5 ötödike +6 hatodika +7 hetedike +8 nyolcadika +9 kilencedike +10 tizedike +11 tizenegyedike +12 tizenkettedike +13 tizenharmadika +14 tizennegyedike +15 tizenötödike +16 tizenhatodika +17 tizenhetedike +18 tizennyolcadika +19 tizenkilencedike +20 huszadika +21 huszonegyedike +22 huszonkettedike +23 huszonharmadika +24 huszonnegyedike +25 huszonötödike +26 huszonhatodika +27 huszonhetedike +28 huszonnyolcadika +29 huszonkilencedike +30 harmincadika +31 harmincegyedike diff --git a/tn/hungarian/data/dates/month_abbr.tsv b/tn/hungarian/data/dates/month_abbr.tsv new file mode 100644 index 0000000..c1ea254 --- /dev/null +++ b/tn/hungarian/data/dates/month_abbr.tsv @@ -0,0 +1,12 @@ +jan január +febr február +márc március +ápr április +máj május +jún június +júl július +aug augusztus +szept szeptember +okt október +nov november +dec december \ No newline at end of file diff --git a/tn/hungarian/data/dates/months.tsv b/tn/hungarian/data/dates/months.tsv new file mode 100644 index 0000000..47a659e --- /dev/null +++ b/tn/hungarian/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 január +2 február +3 március +4 április +5 május +6 június +7 július +8 augusztus +9 szeptember +10 október +11 november +12 december \ No newline at end of file diff --git a/tn/hungarian/data/dates/months_roman.tsv b/tn/hungarian/data/dates/months_roman.tsv new file mode 100644 index 0000000..e48ed60 --- /dev/null +++ b/tn/hungarian/data/dates/months_roman.tsv @@ -0,0 +1,12 @@ +i január +ii február +iii március +iv április +v május +vi június +vii július +viii augusztus +ix szeptember +x október +xi november +xii december \ No newline at end of file diff --git a/tn/hungarian/data/electronic/domain.tsv b/tn/hungarian/data/electronic/domain.tsv new file mode 100644 index 0000000..8955be4 --- /dev/null +++ b/tn/hungarian/data/electronic/domain.tsv @@ -0,0 +1,15 @@ +.com pont com +.org pont org +.gov pont gov +.uk pont u k +.fr pont f r +.net pont net +.br pont b r +.in pont i n +.ru pont r u +.de pont d e +.it pont i t +.es pont e s +.mx pont m x +.us pont u s +.hu pont hu diff --git a/tn/hungarian/data/electronic/server_name.tsv b/tn/hungarian/data/electronic/server_name.tsv new file mode 100644 index 0000000..7bf1f2b --- /dev/null +++ b/tn/hungarian/data/electronic/server_name.tsv @@ -0,0 +1,15 @@ +gmail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast +origo +index +blikk +startlap \ No newline at end of file diff --git a/tn/hungarian/data/electronic/symbols.tsv b/tn/hungarian/data/electronic/symbols.tsv new file mode 100644 index 0000000..ef44fdd --- /dev/null +++ b/tn/hungarian/data/electronic/symbols.tsv @@ -0,0 +1,23 @@ +. pont +- kötőjel +_ aláhúzásjel +! felkiáltójel +# kettőskereszt +$ dollárjel +% százalékjel +& ésjel +' aposztróf +* csillagjel ++ pluszjel +/ perjel +\ fordított perjel += egyenlőjel +? kérdőjel +^ kúpos ékezet +` tompa ékezet +{ nyitó kapcsos zárójel +| függőleges vonal +} berekesztő kapcsos zárójel +~ hullámvonal +, vessző +: kettőspont diff --git a/tn/hungarian/data/fractions/fraction_symbols.tsv b/tn/hungarian/data/fractions/fraction_symbols.tsv new file mode 100644 index 0000000..d0bb922 --- /dev/null +++ b/tn/hungarian/data/fractions/fraction_symbols.tsv @@ -0,0 +1,19 @@ +⅓ 1/3 +⅔ 2/3 +⅕ 1/5 +⅖ 2/5 +⅗ 3/5 +⅘ 4/5 +⅙ 1/6 +⅚ 5/6 +⅐ 1/7 +⅛ 1/8 +⅜ 3/8 +⅝ 5/8 +⅞ 7/8 +⅑ 1/9 +⅒ 1/10 +¼ 1/4 +½ 1/2 +¾ 3/4 +↉ 0/3 diff --git a/tn/hungarian/data/inflection/endings.tsv b/tn/hungarian/data/inflection/endings.tsv new file mode 100644 index 0000000..7d23e89 --- /dev/null +++ b/tn/hungarian/data/inflection/endings.tsv @@ -0,0 +1,51 @@ +a át ának ával áért ává áig aként ában án ánál ába ára ához ából áról ától +ab ot nak bal ért bá ig ként ban on nál ba ra hoz ból ról tól +ág ot nak gal ért gá ig ként ban on nál ba ra hoz ból ról tól +ak at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ák at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +alék ot nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ály lyt lynak llyal lyért llyá lyig lyként lyban lyon lynál lyba lyra lyhoz lyból lyról lytól +amm ot nak al ért á ig ként ban on nál ba ra hoz ból ról tól +am ot nak mal ért má ig ként ban on nál ba ra hoz ból ról tól +ány nyt nynak nnyal nyért nnyá nyig nyként nyban nyon nynál nyba nyra nyhoz nyból nyról nytól +árd ot nak dal ért dá ig ként ban on nál ba ra hoz ból ról tól +ár t nak ral ért rá ig ként ban on nál ba ra hoz ból ról tól +á t nak val ért vá ig ként ban n nál ba ra hoz ból ról tól +at ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +é ét ének ével éért évé éig éként ében én énél ébe ére éhez éből éről étől +ef et nek fel ért fé ig ként ben en nél be re hez ből ről től +ég et nek gel ért gé ig ként ben en nél be re hez ből ről től +ek et nek kel ért ké ig ként ben en nél be re hez ből ről től +ék et nek kel ért ké ig ként ben en nél be re hez ből ről től +ény nyt nynek nnyel nyért nnyé nyig nyként nyben nyen nynél nybe nyre nyhez nyből nyről nytől +erc et nek cel ért cé ig ként ben en nél be re hez ből ről től +er t nek rel ért ré ig ként ben en nél be re hez ből ről től +ertz et nek cel ért cé ig ként ben en nél be re hez ből ről től +est et nek tel ért té ig ként ben en nél be re hez ből ről től +es t nek sel ért sé ig ként ben en nél be re hez ből ről től +és t nek sel ért sé ig ként ben en nél be re hez ből ről től +et et nek tel ért té ig ként ben en nél be re hez ből ről től +é t nek vel ért vé ig ként ben n nél be re hez ből ről től +evek et nek kel ért ké ig ként ben en nél be re hez ből ről től +év evet évnek évvel évért évvé évig évként évben éven évnél évbe évre évhez évből évről évtől +int et nek tel ért té ig ként ben en nél be re hez ből ről től +lyok at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +méteres et t nek sel ért sé ig ként ben en nél be re hez ből ről től +nyek et nek kel ért ké ig ként ben en nél be re hez ből ről től +nyok at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ok at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ók at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +ők et nek kel ért ké ig ként ben ön nél be re höz ből ről től +ók ot nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +on t nak nal ért ná ig ként ban on nál ba ra hoz ból ról tól +ont ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +orint ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +or t nak ral ért rá ig ként ban on nál ba ra hoz ból ról tól +ort ot nak tal ért tá ig ként ban on nál ba ra hoz ból ról tól +os at t nak sal ért sá ig ként ban on nál ba ra hoz ból ról tól +ó t nak val ért vá ig ként ban n nál ba ra hoz ból ról tól +ő t nek vel ért vé ig ként ben n nél be re höz ből ről től +um ot nak mal ért má ig ként ban on nál ba ra hoz ból ról tól +us t nak sal ért sá ig ként ban on nál ba ra hoz ból ról tól +utak at nak kal ért ká ig ként ban on nál ba ra hoz ból ról tól +út utat útnak úttal útért úttá útig útként útban úton útnál útba útra úthoz útból útról úttól diff --git a/tn/hungarian/data/inflection/plural_endings.tsv b/tn/hungarian/data/inflection/plural_endings.tsv new file mode 100644 index 0000000..2c73ccc --- /dev/null +++ b/tn/hungarian/data/inflection/plural_endings.tsv @@ -0,0 +1,39 @@ +er ek +erc ek +ertz ek +amm ok +um ok +alék ok +óra ák +or ok +méteres ek +es ek +és ek +ég ek +ő k ek +orint ok +ók ok +on ok +a ák +ort ok +ont ok +ény nyek +ály lyok +ány nyok +et ek +est ek +ág ok +at ok +ab ok +am ok +év evek +ef ek +int ek +os ak +ó k +us ok +árd ok +ár ok +út utak +é k +á k diff --git a/tn/hungarian/data/inflection/word_endings.tsv b/tn/hungarian/data/inflection/word_endings.tsv new file mode 100644 index 0000000..1d49e04 --- /dev/null +++ b/tn/hungarian/data/inflection/word_endings.tsv @@ -0,0 +1 @@ +óra át ának ával áért ává áig aként ában án ánál ába ára ához ából áról ától akor diff --git a/tn/hungarian/data/measures/measurements.tsv b/tn/hungarian/data/measures/measurements.tsv new file mode 100644 index 0000000..62b2a47 --- /dev/null +++ b/tn/hungarian/data/measures/measurements.tsv @@ -0,0 +1,52 @@ +cm centiméter +ó óra +g gramm +kg kilogramm +μg mikrogramm +mg milligramm +ng nanogramm +pg pikogramm +fg femtogramm +ag attogramm +km² négyzetkilométer +km2 négyzetkilométer +L liter +l liter +dl deciliter +dL deciliter +cl centiliter +cL centiliter +ml milliliter +mL milliliter +μl mikroliter +μL mikroliter +nl nanoliter +nL nanoliter +fl femtoliter +fL femtoliter +m méter +dm deciméter +cm centiméter +mm milliméter +dkm dekaméter +hm hektométer +km kilométer +mrm miriaméter +µm mikrométer +nm nanométer +pm pikométer +fm femtométer +am attométer +zm zeptométer +ym yoktométer +m² négyzetméter +m2 négyzetméter +m³ köbméter +m3 köbméter +mph mérföld per óra +mi/h mérföld per óra +ml milliliter +mm milliméter +ms milliszekundum +% százalék +mp másodperc \ No newline at end of file diff --git a/tn/hungarian/data/money/alphabet.tsv b/tn/hungarian/data/money/alphabet.tsv new file mode 100644 index 0000000..9c2274c --- /dev/null +++ b/tn/hungarian/data/money/alphabet.tsv @@ -0,0 +1,44 @@ +A a +Á á +B bé +C cé +Cs csé +D dé +Dz dzé +Dzs dzsé +E e +É é +F ef +G gé +Gy gyé +H há +I i +Í í +J jé +K ká +L el +Ly elly +M em +N en +Ny eny +O o +Ó ó +Ö ö +Ő ő +P pé +Q kú +R er +S es +Sz esz +T té +Ty tyé +U u +Ú ú +Ü ü +Ű ű +V vé +W dupla vé +X iksz +Y ipszilon +Z zé +Zs zsé diff --git a/tn/hungarian/data/money/currency.tsv b/tn/hungarian/data/money/currency.tsv new file mode 100644 index 0000000..0354dbc --- /dev/null +++ b/tn/hungarian/data/money/currency.tsv @@ -0,0 +1,11 @@ +SEK svéd korona +GBP angol font +£ font +USD amerikai dollár +US$ amerikai dollár +$ dollár +IR£ ír font +IEP ír font +NOK norvég korona +HUF magyar forint +Ft forint diff --git a/tn/hungarian/data/money/currency_minor.tsv b/tn/hungarian/data/money/currency_minor.tsv new file mode 100644 index 0000000..b222ea2 --- /dev/null +++ b/tn/hungarian/data/money/currency_minor.tsv @@ -0,0 +1,11 @@ +SEK öre +GBP penny +USD cent +US$ cent +$ cent +IR£ penny +£ penny +IEP penny +NOK öre +HUF fillér +Ft fillér diff --git a/tn/hungarian/data/number/digit.tsv b/tn/hungarian/data/number/digit.tsv new file mode 100644 index 0000000..4254fd0 --- /dev/null +++ b/tn/hungarian/data/number/digit.tsv @@ -0,0 +1,9 @@ +egy 1 +kettő 2 +három 3 +négy 4 +öt 5 +hat 6 +hét 7 +nyolc 8 +kilenc 9 diff --git a/tn/hungarian/data/number/digit_inline.tsv b/tn/hungarian/data/number/digit_inline.tsv new file mode 100644 index 0000000..2336ab3 --- /dev/null +++ b/tn/hungarian/data/number/digit_inline.tsv @@ -0,0 +1,9 @@ +egy 1 +két 2 +három 3 +négy 4 +öt 5 +hat 6 +hét 7 +nyolc 8 +kilenc 9 diff --git a/tn/hungarian/data/number/quantities.tsv b/tn/hungarian/data/number/quantities.tsv new file mode 100644 index 0000000..7229518 --- /dev/null +++ b/tn/hungarian/data/number/quantities.tsv @@ -0,0 +1,21 @@ +mrd milliárd +millió +milliárd +billió +billiárd +trillió +trilliárd +kvadrillió +kvadrilliárd +kvintillió +kvintilliárd +szextillió +decillió +decilliárd +undecillió +undecilliárd +duodecillió +tridecillió +kvadecillió +kvintdecillió +szexdecillió diff --git a/tn/hungarian/data/number/tens.tsv b/tn/hungarian/data/number/tens.tsv new file mode 100644 index 0000000..2471faa --- /dev/null +++ b/tn/hungarian/data/number/tens.tsv @@ -0,0 +1,9 @@ +tíz 10 +húsz 20 +harminc 30 +negyven 40 +ötven 50 +hatvan 60 +hetven 70 +nyolcvan 80 +kilencven 90 diff --git a/tn/hungarian/data/number/tens_inline.tsv b/tn/hungarian/data/number/tens_inline.tsv new file mode 100644 index 0000000..6f3a07a --- /dev/null +++ b/tn/hungarian/data/number/tens_inline.tsv @@ -0,0 +1,9 @@ +tizen 1 +huszon 2 +harminc 3 +negyven 4 +ötven 5 +hatvan 6 +hetven 7 +nyolcvan 8 +kilencven 9 diff --git a/tn/hungarian/data/number/zero.tsv b/tn/hungarian/data/number/zero.tsv new file mode 100644 index 0000000..a964ff4 --- /dev/null +++ b/tn/hungarian/data/number/zero.tsv @@ -0,0 +1 @@ +nulla 0 diff --git a/tn/hungarian/data/ordinals/endings.tsv b/tn/hungarian/data/ordinals/endings.tsv new file mode 100644 index 0000000..cda9bab --- /dev/null +++ b/tn/hungarian/data/ordinals/endings.tsv @@ -0,0 +1,23 @@ +kilenc kilencedik +nyolc nyolcadik +hét hetedik +hat hatodik +öt ötödik +négy negyedik +három harmadik +kettő kettedik +két kettedik +tíz tizedik +egy egyedik +húsz huszadik +harminc harmincadik +ven venedik +van vanadik +száz századik +ezer ezredik +millió milliomodik +milliárd milliárdodik +billió billiomodik +billiárd billiárdodik +trillió trilliomodik +trilliárd trilliárdodik diff --git a/tn/hungarian/data/ordinals/exceptional.tsv b/tn/hungarian/data/ordinals/exceptional.tsv new file mode 100644 index 0000000..c9cedc0 --- /dev/null +++ b/tn/hungarian/data/ordinals/exceptional.tsv @@ -0,0 +1,4 @@ +egy első +kettő második +két második +nulla nulladik diff --git a/tn/hungarian/data/telephone/area_codes.tsv b/tn/hungarian/data/telephone/area_codes.tsv new file mode 100644 index 0000000..040dc0c --- /dev/null +++ b/tn/hungarian/data/telephone/area_codes.tsv @@ -0,0 +1,71 @@ +1 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +40 +41 +42 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +59 +60 +61 +62 +63 +66 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +99 diff --git a/tn/hungarian/data/telephone/country_codes.tsv b/tn/hungarian/data/telephone/country_codes.tsv new file mode 100644 index 0000000..35aba1b --- /dev/null +++ b/tn/hungarian/data/telephone/country_codes.tsv @@ -0,0 +1,213 @@ +1 +20 +211 +212 +213 +216 +218 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +27 +290 +291 +297 +298 +299 +30 +31 +32 +33 +34 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +36 +370 +371 +372 +373 +374 +375 +376 +377 +378 +380 +381 +382 +383 +385 +386 +387 +389 +39 +40 +41 +420 +421 +423 +43 +44 +45 +46 +47 +48 +49 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +51 +52 +53 +54 +55 +56 +57 +58 +590 +591 +592 +593 +594 +595 +596 +597 +598 +60 +61 +62 +63 +64 +65 +66 +670 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +685 +686 +687 +688 +689 +690 +691 +692 +7 +800 +808 +81 +82 +84 +850 +852 +853 +855 +856 +86 +870 +878 +880 +881 +882 +886 +90 +91 +92 +93 +94 +95 +960 +961 +962 +963 +964 +965 +966 +967 +968 +970 +971 +972 +973 +974 +975 +976 +977 +979 +98 +992 +993 +994 +995 +996 +997 +998 diff --git a/tn/hungarian/data/telephone/telephone_abbr.tsv b/tn/hungarian/data/telephone/telephone_abbr.tsv new file mode 100644 index 0000000..0a4d64c --- /dev/null +++ b/tn/hungarian/data/telephone/telephone_abbr.tsv @@ -0,0 +1,3 @@ +tel. telefon +tel telefon +fax diff --git a/tn/hungarian/data/time/time_zone.tsv b/tn/hungarian/data/time/time_zone.tsv new file mode 100644 index 0000000..b53395b --- /dev/null +++ b/tn/hungarian/data/time/time_zone.tsv @@ -0,0 +1,9 @@ +cst c s t +cet c e t +cest c e s t +pst p s t +est e s t +pt p t +et e t +gmt g m t +bst b s t diff --git a/tn/hungarian/data/whitelist.tsv b/tn/hungarian/data/whitelist.tsv new file mode 100644 index 0000000..c0b2975 --- /dev/null +++ b/tn/hungarian/data/whitelist.tsv @@ -0,0 +1,24 @@ +stb. s a többi +vs. versus +vö. vesd össze +vmi valami +vki valaki +ún. úgynevezett +uő ugyanő +uo. ugyanott +ui. ugyanis +ua. ugyanaz +ti. tudniillik +szül. született +megh. meghalt +sztem szerintem +Szt. Szent +pu. pályaudvar +pl. például +ált. általános +és tsai. és társai +ford. fordította +i. m. idézett mű +ifj. ifjabb +kb. körülbelül +ld. lásd diff --git a/tn/hungarian/data/whitelist_inflect.tsv b/tn/hungarian/data/whitelist_inflect.tsv new file mode 100644 index 0000000..bdae29a --- /dev/null +++ b/tn/hungarian/data/whitelist_inflect.tsv @@ -0,0 +1,39 @@ +m-es méteres +Hz hertz +MHz megahertz +mp másodperc +Mrd milliárd +op. opus +Ft-os forintos +gimn. gimnázium +hsz. határozószó +isk. iskola +ism. ismétlés +ker. kerület +kg-os kilogrammos +kir. király +km-es kilométeres +korm. kormány +köv. következő +Kr. Krisztus +krt. körút +ktg. költség +max. maximum +dkg dekagramm +db darab +em. emelet +évf. évfolyam +fej. fejezet +fn. főnév +fszt. földszint +Ft forint +cm-es centiméteres +cs. csoport +dr. doktor +prof. professzor +u. utca +tv. törvény +tel. telefon +Pf. postafiók +alez. alezredes +bek. bekezdés diff --git a/tn/hungarian/data/whitelist_inflect_sg.tsv b/tn/hungarian/data/whitelist_inflect_sg.tsv new file mode 100644 index 0000000..e2d689a --- /dev/null +++ b/tn/hungarian/data/whitelist_inflect_sg.tsv @@ -0,0 +1,14 @@ +Bp. Budapest +Magyaro. Magyarország +D-DK dél-délkelet +DK délkelet +D-DNy dél-délnyugat +DNy délnyugat +É-ÉK észak-északkelet +ÉK északkelet +É-ÉNy észak-északnyugat +ÉNy északnyugat +K-DK kelet-délkelet +K-ÉK kelet-északkelet +Ny-DNy nyugat-délnyugat +Ny-ÉNy nyugat-északnyugat \ No newline at end of file diff --git a/tn/hungarian/normalizer.py b/tn/hungarian/normalizer.py new file mode 100644 index 0000000..5b7c9ad --- /dev/null +++ b/tn/hungarian/normalizer.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from tn.hungarian.rules import date, electronic, money, telephone, time +from tn.hungarian.rules.cardinal import Cardinal +from tn.hungarian.rules.date import Date +from tn.hungarian.rules.decimal import Decimal +from tn.hungarian.rules.electronic import Electronic +from tn.hungarian.rules.fraction import Fraction +from tn.hungarian.rules.measure import Measure +from tn.hungarian.rules.money import Money +from tn.hungarian.rules.ordinal import Ordinal +from tn.hungarian.rules.punctuation import Punctuation +from tn.hungarian.rules.telephone import Telephone +from tn.hungarian.rules.time import Time +from tn.hungarian.rules.whitelist import WhiteList +from tn.hungarian.rules.word import Word +from tn.processor import Processor, RuleSpec + +TOKEN_ORDERS = {} +for _module in (date, electronic, money, telephone, time): + TOKEN_ORDERS.update(_module.TOKEN_ORDER) + + +class Normalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="hu_normalizer", token_orders=TOKEN_ORDERS) + self.build_fst("hu_tn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + ordinal = Ordinal(cardinal) + decimal = Decimal(cardinal) + fraction = Fraction(cardinal, ordinal) + punctuation = Punctuation() + word = Word(punctuation) + rules = ( + RuleSpec(WhiteList(), 1.01), + RuleSpec(Time(cardinal), 1.09), + RuleSpec(Measure(cardinal, decimal, fraction), 1.08), + RuleSpec(cardinal, 1.1), + RuleSpec(fraction, 1.09), + RuleSpec(Date(cardinal), 1.1), + RuleSpec(ordinal, 1.1), + RuleSpec(decimal, 1.1), + RuleSpec(Money(cardinal, decimal), 1.1), + # Slightly above cardinal: "1 034 068" is also a valid local number. + RuleSpec(Telephone(cardinal), 1.105), + RuleSpec(Electronic(), 1.11), + RuleSpec(word, 200, verbalize=False), + RuleSpec(punctuation, verbalize=False), + ) + classify = self.tagger_union(rules) + # Classified tokens are separated by spaces or punctuation; punctuation + # may stick to either side of a token. + space = self.DELETE_EXTRA_SPACE + punct = pynutil.add_weight(punctuation.tagger, 2.1) + punct = (punct + (space.ques + punct).star).optimize() + separator = space | space.ques + punct + space.ques + tokens = (punct + space.ques).ques + classify + (separator + classify).star + (space.ques + punct).ques + self.tagger = self.DELETE_SPACE + (tokens | punct) + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + self.verbalizer = self.join_tokens( + self.verbalizer_union(rules), (word.verbalizer, punctuation.verbalizer), + opening=self.OPENING_MARKS + ("-",), closing=self.CLOSING_MARKS + ("-",)) diff --git a/tn/hungarian/rules/__init__.py b/tn/hungarian/rules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tn/hungarian/rules/cardinal.py b/tn/hungarian/rules/cardinal.py new file mode 100644 index 0000000..fca9f29 --- /dev/null +++ b/tn/hungarian/rules/cardinal.py @@ -0,0 +1,98 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import DIGIT, NONZERO, data_map +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Long-scale powers above a thousand, lowest first. +POWERS = ["millió", "milliárd", "billió", "billiárd", "trillió", "trilliárd"] + + +class Cardinal(Processor): + """Cardinals, e.g. ``2010`` -> ``kettőezer-tíz``, ``-1 000`` -> ``mínusz ezer``. + + The tagger keeps the written digits (with optional ``.``/space group + separators); the verbalizer reads them. Numbers have up to 24 digits. + """ + + def __init__(self): + super().__init__("cardinal", token_orders=TOKEN_ORDER) + self.build_numbers() + self.sign = pynini.cross("-", "mínusz") + self.sign_tagger = (self.tag_field("negative", self.sign) + insert(" ")).ques + self.sign_verbalizer = (self.verbalize_field("negative", self.sign) + pynini.accep(" ")).ques + self.tagger = self.add_tokens(self.sign_tagger + self.tag_field("integer", self.graph)) + self.verbalizer = self.delete_tokens(self.sign_verbalizer + self.verbalize_field("integer", self.graph)) + + def build_numbers(self): + digit = data_map("number/digit.tsv", invert=True) + digit_inline = data_map("number/digit_inline.tsv", invert=True) + tens = data_map("number/tens.tsv", invert=True) + tens_inline = data_map("number/tens_inline.tsv", invert=True) + self.zero = data_map("number/zero.tsv", invert=True) + self.digit = digit + digit_no_one = (NONZERO - "1") @ digit + digit_inline_no_one = (NONZERO - "1") @ digit_inline + + graph_tens = (tens_inline + digit | tens).optimize() + self.tens = graph_tens + self.two_digit_non_zero = (digit | graph_tens | delete("0") + digit).optimize() + base_hundreds = pynini.cross("1", "száz") | digit_inline_no_one + insert("száz") + hundreds = (base_hundreds + (delete("00") | graph_tens | delete("0") + digit)).optimize() + self.hundreds = hundreds + # Exactly three digits, not all zero. + comp = (hundreds | delete("0") + graph_tens | delete("00") + digit).optimize() + comp_no_one = (hundreds | delete("0") + graph_tens | delete("00") + digit_no_one).optimize() + # One to three digits without a leading zero. + head = (digit | graph_tens | hundreds).optimize() + head_no_one = (digit_no_one | graph_tens | hundreds).optimize() + self.up_to_hundreds = head + + hyphen = insert("-") + # Six digits, not all zero. + level = ( + delete("000") + comp + | comp_no_one + insert("ezer") + (hyphen + comp | delete("000")) + | delete("001") + insert("ezer") + (comp | delete("000")) + ).optimize() + top = head | ( + head_no_one + insert("ezer") + (hyphen + comp | delete("000")) + | pynini.cross("1", "ezer") + (comp | delete("000")) + ) + width = 6 + for word in POWERS: + rest = hyphen + level | delete("0" * width) + top |= (pynini.cross("1", word) | head_no_one + insert(word)) + rest + level = ( + delete("000") + level | (pynini.cross("001", word) | comp_no_one + insert(word)) + rest + ).optimize() + width += 3 + self.plain = (self.zero | top).optimize() + + separator = delete(pynini.union(".", " ")) + grouped = NONZERO + pynini.closure(DIGIT, 0, 2) + (separator + DIGIT ** 3).plus + self.graph = ((NONZERO + DIGIT.star | grouped) @ self.plain | self.zero).optimize() + + zero_space = self.zero + insert(" ") + self.two_digits_read = ((NONZERO + DIGIT) @ graph_tens | zero_space + digit).optimize() + self.three_digits_read = ( + hundreds | zero_space + graph_tens | zero_space + zero_space + digit + ).optimize() + self.four_digits_read = ((NONZERO + DIGIT ** 3) @ self.plain | zero_space + self.three_digits_read).optimize() diff --git a/tn/hungarian/rules/date.py b/tn/hungarian/rules/date.py new file mode 100644 index 0000000..9311436 --- /dev/null +++ b/tn/hungarian/rules/date.py @@ -0,0 +1,104 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete + +from tn.hungarian.rules.utils import DIGIT, NONZERO, SIGMA, TO_LOWER, TO_UPPER, data_labels, data_map +from tn.processor import Processor + +TOKEN_ORDER = {"date": ["year", "month", "day"]} + +CASE_ENDINGS = { + "e": "ét ének ével éért évé éig eként éül ében én énél ébe ére éhez éből éről étől", + "a": "át ának ával áért ává áig aként ául ában án ánál ába ára ához ából áról ától", +} + + +def suffixed_days(days): + """Day numbers with Hungarian case endings, e.g. ``1-jén`` -> ``elsején``.""" + + output = [] + for number, day in days: + for ending in ["je", "a", "e"]: + if day.endswith(ending): + output.append(("{}-{}".format(number, ending), day)) + break + base = day[:-1] + for ending in CASE_ENDINGS[day[-1]].split(" "): + if day.endswith("eje"): + output.append(("{}-j{}".format(number, ending), base + ending)) + output.append(("{}-{}".format(number, ending), base + ending)) + else: + output.append(("{}-{}".format(number, ending), base + ending)) + output.append(("{}-{}".format(number, ending[1:]), base + ending)) + # Adjectival "-i" forms, e.g. "1-i" -> "elsei". + output.append(("{}-i".format(number), base + "i")) + if day == "elseje": + output.append(("{}-ji".format(number), base + "i")) + output.append(("{}-{}i".format(number, day[-1]), day + "i")) + return output + + +class Date(Processor): + """Dates, e.g. ``2010. ápr. 1-jén`` -> ``kettőezer-tíz április elsején``. + + Raw ``year``/``month``/``day`` fields; date dots and dashes are separators + deleted by the tagger. Fields are verbalized in year-month-day order. + """ + + def __init__(self, cardinal): + super().__init__("date", token_orders=TOKEN_ORDER) + days = [(row[0], row[1]) for row in data_labels("dates/days.tsv")] + optional_zero = delete("0").ques + NONZERO | NONZERO + DIGIT + day_number = (optional_zero @ data_map("dates/days.tsv")).optimize() + day_suffixed = pynini.string_map(suffixed_days(days)) + day_words = pynini.project(day_suffixed, "output") | pynini.project(day_number, "output") + self.day = (day_number | day_suffixed | day_words).optimize() + + months = data_map("dates/months.tsv") + month_number = (optional_zero @ months).optimize() + names = pynini.project(months, "output") + month_name = (names | (TO_LOWER + SIGMA) @ names).optimize() + abbr = data_map("dates/month_abbr.tsv") + roman = data_map("dates/months_roman.tsv") + roman |= pynini.invert(pynini.invert(roman) @ pynini.closure(TO_UPPER)) + self.month = (month_number | month_name | abbr | roman).optimize() + self.year = ((NONZERO + pynini.closure(DIGIT, 1, 3)) @ cardinal.plain).optimize() + + dot = delete(".") + space = pynini.accep(" ") + dash = pynini.cross("-", " ") + year_t = self.tag_field("year", self.year) + month_t = ( + self.tag_field("month", abbr | roman | month_number) + dot + | self.tag_field("month", month_name) + ) + day_t = self.tag_field("day", day_number) + dot.ques | self.tag_field("day", day_suffixed | day_words) + month_number_t = self.tag_field("month", month_number) + day_number_t = self.tag_field("day", day_number) + graph = (year_t + dot + space).ques + month_t + space + day_t + graph |= (year_t + dash).ques + month_number_t + dash + day_number_t + graph |= year_t + dot + space + month_t + graph |= day_number_t + dash + month_number_t + (dash + year_t).ques + for separator in [".", "/"]: + separator = pynini.cross(separator, " ") + graph |= day_number_t + separator + month_number_t + separator + year_t + self.tagger = self.add_tokens(graph) + + year_v = self.verbalize_field("year", self.year) + space + month_v = self.verbalize_field("month", self.month) + day_v = space + self.verbalize_field("day", self.day) + self.verbalizer = self.delete_tokens(year_v.ques + month_v + day_v.ques) diff --git a/tn/hungarian/rules/decimal.py b/tn/hungarian/rules/decimal.py new file mode 100644 index 0000000..84611d2 --- /dev/null +++ b/tn/hungarian/rules/decimal.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import DIGIT, NONZERO, data_labels, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Names of the fractional positions after "tized" and "század". +DECIMAL_NAMES = ["ezred", "milliomod", "milliárdod", "billiomod", "billiárdod", "trilliomod", "trilliárdod"] + + +def quantities(): + """Quantity words with their inflected and abbreviated forms, e.g. ``mrd``.""" + + output = [] + for row in data_labels("number/quantities.tsv"): + if len(row) == 2: + output.append((row[0], row[1])) + output += naive_inflector(row[0], row[1], True) + else: + output.append((row[0], row[0])) + real = [form[1] for form in naive_inflector(".", row[0], True)] + output += [(form, form) for form in real] + if "lli" in row[0]: + output.append((row[0].replace("lli", "li"), row[0])) + output += [(form.replace("lli", "li"), form) for form in real] + return pynini.string_map(output).optimize() + + +class Decimal(Processor): + """Decimals with a comma, e.g. ``11,04006`` -> ``tizenegy egész négyezer-hat százezred``. + + Fields stay raw: ``integer_part``, ``fractional_part`` and an optional + ``quantity`` (``1 mrd`` -> ``egy milliárd``). + """ + + def __init__(self, cardinal): + super().__init__("decimal", token_orders=TOKEN_ORDER) + names = ["tized", "század"] + [prefix + name for name in DECIMAL_NAMES for prefix in ["", "tíz", "száz"]] + fractional = None + leading = pynini.accep("") + for name in names: + # The last non-zero digit fixes the denominator; trailing zeros are dropped. + number = (leading + NONZERO) @ (delete("0").star + cardinal.plain) + leading += DIGIT + branch = number + delete("0").star + insert(" " + name) + fractional = branch if fractional is None else fractional | branch + self.fractional = fractional.optimize() + self.integer = cardinal.graph + self.quantity = quantities() + self.quantity_integer = cardinal.up_to_hundreds + self.sign = cardinal.sign + + sign_t = cardinal.sign_tagger + integer_t = self.tag_field("integer_part", self.integer) + fractional_t = self.tag_field("fractional_part", self.fractional) + quantity_t = pynini.accep(" ") + self.tag_field("quantity", self.quantity) + decimal_t = integer_t + delete(",") + insert(" ") + fractional_t + self.unsigned_tagger = (decimal_t + quantity_t.ques | self.tag_field("integer_part", self.quantity_integer) + + quantity_t).optimize() + self.tagger = self.add_tokens(sign_t + self.unsigned_tagger) + + integer_v = self.verbalize_field("integer_part", self.integer) + fractional_v = self.verbalize_field("fractional_part", self.fractional) + quantity_v = pynini.accep(" ") + self.verbalize_field("quantity", self.quantity) + self.unsigned_verbalizer = (integer_v + pynini.cross(" ", " egész ") + fractional_v + quantity_v.ques + | integer_v + quantity_v).optimize() + self.signed_verbalizer = cardinal.sign_verbalizer + self.unsigned_verbalizer + self.verbalizer = self.delete_tokens(self.signed_verbalizer) diff --git a/tn/hungarian/rules/electronic.py b/tn/hungarian/rules/electronic.py new file mode 100644 index 0000000..a05e1cd --- /dev/null +++ b/tn/hungarian/rules/electronic.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import byte +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.hungarian.rules.utils import data_keys, data_map +from tn.processor import Processor + +TOKEN_ORDER = {"electronic": ["protocol", "username", "domain"]} + + +class Electronic(Processor): + """E-mails, URLs and handles, e.g. ``abc@def.hu`` -> ``a b c kukac d e f pont hu``. + + Raw ``protocol``/``username``/``domain`` fields; the verbalizer spells + them, reading symbols, digits, known servers and domains as words. + """ + + def __init__(self): + super().__init__("electronic", token_orders=TOKEN_ORDER) + symbols = data_map("electronic/symbols.tsv") + symbol_chars = data_keys("electronic/symbols.tsv") + alnum = byte.ALPHA | byte.DIGIT + accepted = (alnum | pynini.difference(symbol_chars, ".")).plus + all_chars = (alnum | symbol_chars).plus + domain = accepted + ("." + accepted).plus + protocol_start = pynini.union("https://", "http://") + protocol = protocol_start | "www." | protocol_start + "www." + + digits = data_map("number/digit.tsv", invert=True) | data_map("number/zero.tsv", invert=True) + spelled_char = symbols | digits | byte.ALPHA + + def spell(graph): + return (graph @ (spelled_char + (insert(" ") + spelled_char).star)).optimize() + + known = data_map("electronic/domain.tsv") | data_keys("electronic/server_name.tsv") + piece = add_weight(spelled_char, 0.0001) | known + terminal_dot = pynini.cross(".", " .").ques + self.domain = ((domain @ (piece + (insert(" ") + piece).star)) + terminal_dot).optimize() + self.username = spell(all_chars) + self.handle = spell(accepted + ("." + accepted).star) + self.protocol = spell(protocol) + + domain_t = self.tag_field("domain", self.domain) + email_t = self.tag_field("username", self.username) + pynini.cross("@", " ") + domain_t + handle_t = delete("@") + self.tag_field("username", self.handle) + url_t = self.tag_field("protocol", self.protocol) + insert(" ") + domain_t + self.tagger = self.add_tokens(domain_t | email_t | handle_t | url_t) + + domain_v = self.verbalize_field("domain", self.domain) + username_v = self.verbalize_field("username", self.username) + email_v = username_v + pynini.cross(" ", " kukac ") + domain_v + handle_v = insert("kukac ") + username_v + url_v = self.verbalize_field("protocol", self.protocol) + pynini.accep(" ") + domain_v + self.verbalizer = self.delete_tokens(domain_v | email_v | handle_v | url_v) diff --git a/tn/hungarian/rules/fraction.py b/tn/hungarian/rules/fraction.py new file mode 100644 index 0000000..97dd36d --- /dev/null +++ b/tn/hungarian/rules/fraction.py @@ -0,0 +1,55 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import data_map +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Fraction(Processor): + """Fractions, e.g. ``2/3`` -> ``kettő harmad``, ``1 1/2`` -> ``egy és egy fél``. + + Raw fields: ``integer_part``, ``numerator``, ``denominator`` or a + vulgar fraction ``symbol`` such as ``⅔``. + """ + + def __init__(self, cardinal, ordinal): + super().__init__("fraction", token_orders=TOKEN_ORDER) + numerator = cardinal.graph + denominator = ordinal.fractional + self.symbol = (data_map("fractions/fraction_symbols.tsv") + @ (numerator + pynini.cross("/", " ") + denominator)).optimize() + + integer_t = self.tag_field("integer_part", cardinal.graph) + pynini.accep(" ") + fraction_t = ( + self.tag_field("numerator", numerator) + delete(pynini.union("/", " / ")) + insert(" ") + + self.tag_field("denominator", denominator) + | self.tag_field("symbol", self.symbol) + ) + self.unsigned_tagger = (integer_t.ques + fraction_t).optimize() + self.tagger = self.add_tokens(cardinal.sign_tagger + self.unsigned_tagger) + + integer_v = self.verbalize_field("integer_part", cardinal.graph) + pynini.cross(" ", " és ") + fraction_v = ( + self.verbalize_field("numerator", numerator) + pynini.accep(" ") + + self.verbalize_field("denominator", denominator) + | self.verbalize_field("symbol", self.symbol) + ) + self.unsigned_verbalizer = (integer_v.ques + fraction_v).optimize() + self.verbalizer = self.delete_tokens(cardinal.sign_verbalizer + self.unsigned_verbalizer) diff --git a/tn/hungarian/rules/measure.py b/tn/hungarian/rules/measure.py new file mode 100644 index 0000000..14a548d --- /dev/null +++ b/tn/hungarian/rules/measure.py @@ -0,0 +1,58 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import data_map, inflected_map +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Measure(Processor): + """Measures, e.g. ``3mm-ben`` -> ``három milliméterben``, ``5 km/h``. + + The number keeps the raw cardinal/decimal/fraction fields and the + (possibly inflected) unit stays raw in ``units``. + """ + + def __init__(self, cardinal, decimal, fraction): + super().__init__("measure", token_orders=TOKEN_ORDER) + unit = (data_map("measures/measurements.tsv") + | inflected_map("measures/measurements.tsv", lower=True)).optimize() + per = pynini.cross("/", "per ") + unit + self.units = (unit + (pynini.cross("", " ") + per).ques | per).optimize() + times = pynini.union("x", "X") + + number_t = ( + self.tag_field("integer", cardinal.graph) + | decimal.unsigned_tagger + | fraction.unsigned_tagger + ) + space_t = delete(" ").ques + insert(" ") + units_t = self.tag_field("units", self.units) + self.tagger = self.add_tokens( + cardinal.sign_tagger + number_t + space_t + units_t + | self.tag_field("integer", cardinal.graph) + insert(" ") + self.tag_field("units", times) + ) + + number_v = ( + self.verbalize_field("integer", cardinal.graph) + | decimal.unsigned_verbalizer + | fraction.unsigned_verbalizer + ) + units_v = pynini.accep(" ") + self.verbalize_field("units", self.units | times) + self.verbalizer = self.delete_tokens(cardinal.sign_verbalizer + number_v + units_v) diff --git a/tn/hungarian/rules/money.py b/tn/hungarian/rules/money.py new file mode 100644 index 0000000..703954e --- /dev/null +++ b/tn/hungarian/rules/money.py @@ -0,0 +1,81 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import (DIGIT, HU_ALPHA, NONZERO, SIGMA, data_labels, inflect_abbreviation, + naive_inflector) +from tn.processor import Processor + +TOKEN_ORDER = {"money": ["integer_part", "fractional_part", "quantity", "currency", "minor"]} + + +class Money(Processor): + """Money, e.g. ``$1,01`` -> ``egy dollár egy cent``, ``1 mrd GBP-t`` -> ``egy milliárd angol fontot``. + + The currency stays raw (including Hungarian case endings such as + ``GBP-t``). A one- or two-digit amount after a leading symbol is tagged + as ``minor`` (read in cents); other fractions are decimal readings. + """ + + def __init__(self, cardinal, decimal): + super().__init__("money", token_orders=TOKEN_ORDER) + majors = [(row[0], row[1]) for row in data_labels("money/currency.tsv")] + minors = dict((row[0], row[1]) for row in data_labels("money/currency_minor.tsv")) + symbols = pynini.union(*[pynini.escape(symbol) for symbol, _ in majors]).optimize() + inflected = [] + for symbol, word in majors: + inflected.append((symbol, word)) + inflected += naive_inflector(symbol, word) + if re.match("^[A-Z]{3}$", symbol): + inflected += inflect_abbreviation(symbol, word) + currency = pynini.string_map(inflected).optimize() + + integer_t = self.tag_field("integer_part", cardinal.graph) + decimal_t = decimal.unsigned_tagger + # After a leading symbol only long fractions or quantities read as decimals. + long_fraction = pynini.union(DIGIT, ".", " ").plus + "," + pynini.closure(DIGIT, 2) + NONZERO + pynini.closure("0") + prefix_decimal = (long_fraction | SIGMA + HU_ALPHA) @ decimal_t + zero_cents_t = self.tag_field("fractional_part", pynini.closure("0", 1)) + minor_t = self.tag_field("minor", DIGIT.ques + NONZERO + pynini.closure("0")) + prefix = self.tag_field("currency", symbols) + insert(" ") + ( + integer_t + (delete(",") + insert(" ") + (zero_cents_t | minor_t)).ques + | prefix_decimal + ) + space = delete(" ").ques + insert(" ") + suffix = (integer_t | decimal_t) + space + self.tag_field("currency", currency) + self.tagger = self.add_tokens(prefix | suffix) + + cents = (DIGIT.ques + NONZERO + delete("0").star) @ ( + delete("0") + NONZERO | NONZERO + insert("0") | NONZERO + DIGIT + ) @ cardinal.two_digit_non_zero + integer_v = self.verbalize_field("integer_part", cardinal.graph) + nonzero_integer = pynini.difference(pynini.project(cardinal.graph, "input"), "0") @ cardinal.graph + nonzero_integer_v = self.verbalize_field("integer_part", nonzero_integer) + zero_cents_v = self.verbalize_field("fractional_part", delete(pynini.closure("0", 1)), leading_space=True) + currency_v = pynini.accep(" ") + self.verbalize_field("currency", currency) + amount_v = (integer_v + zero_cents_v.ques | decimal.unsigned_verbalizer) + currency_v + minor_v = [] + for symbol, word in majors: + minor = self.verbalize_field("minor", cents, leading_space=True) + insert(" " + minors[symbol]) + symbol = pynini.escape(symbol) + currency_symbol_v = self.verbalize_field("currency", pynini.cross(symbol, word)) + minor_v.append(nonzero_integer_v + pynini.accep(" ") + currency_symbol_v + insert(" ") + minor) + minor_v.append(self.verbalize_field("integer_part", delete("0")) + delete(" ") + + self.verbalize_field("currency", delete(symbol)) + minor) + self.verbalizer = self.delete_tokens(amount_v | pynini.union(*minor_v)) diff --git a/tn/hungarian/rules/ordinal.py b/tn/hungarian/rules/ordinal.py new file mode 100644 index 0000000..b20ceb5 --- /dev/null +++ b/tn/hungarian/rules/ordinal.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete + +from tn.hungarian.rules.utils import data_map, suffix_rewrite +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Ordinal(Processor): + """Ordinals written with a trailing dot, e.g. ``31.`` -> ``harmincegyedik``. + + The tagger keeps the digits and drops the ordinal dot; the verbalizer + reads the cardinal and rewrites its last word into the ordinal form. + """ + + def __init__(self, cardinal): + super().__init__("ordinal", token_orders=TOKEN_ORDER) + to_ordinal = suffix_rewrite(data_map("ordinals/endings.tsv"), data_map("ordinals/exceptional.tsv")) + self.bare = (cardinal.plain @ to_ordinal).optimize() + self.graph = (cardinal.graph @ to_ordinal).optimize() + # Denominators: "harmadik" -> "harmad", "második" -> "fél". + exceptions = pynini.string_map([("első", "egyed"), ("második", "fél")]) + self.fractional = (self.bare @ suffix_rewrite(delete("ik"), exceptions)).optimize() + self.tagger = self.add_tokens(self.tag_field("integer", self.graph) + delete(".")) + self.verbalizer = self.delete_tokens(self.verbalize_field("integer", self.graph)) diff --git a/tn/hungarian/rules/punctuation.py b/tn/hungarian/rules/punctuation.py new file mode 100644 index 0000000..f2383e6 --- /dev/null +++ b/tn/hungarian/rules/punctuation.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini +from pynini.lib.pynutil import add_weight + +from tn.processor import Processor + +TOKEN_ORDER = {} + +# Symbols that other rules read aloud are not split off as punctuation. +EXCLUDED = set("&#@§™®©_%*+/=^|~$£€₩¥°º") + + +class Punctuation(Processor): + """Punctuation marks are kept verbatim.""" + + def __init__(self): + super().__init__("punctuation", token_orders=TOKEN_ORDER) + unicode_marks = [chr(i) for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in ("[", "]")] + ascii_marks = list("!#%&'()*+,-./:;<=>?@^_`{|}~\"") + self.punct_marks = [mark for mark in unicode_marks + ["[", "]"] + ascii_marks if mark not in EXCLUDED] + self.punct = pynini.union(*[pynini.escape(mark) for mark in self.punct_marks]).optimize() + whitespace = pynini.union(" ", "\t", "\n", "\r", "\u00a0") + text = (self.VCHAR - whitespace - pynini.union("<", ">")).plus + emphasis = pynini.accep("<") + (text + pynini.accep("/").ques | pynini.accep("/") + text) + ">" + self.graph = (self.punct.plus | emphasis).optimize() + space = add_weight(self.SPACE, -0.0001).star + self.tagger = self.add_tokens(self.tag_field("value", space + self.graph + space)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.SPACE.star + self.graph + self.SPACE.star)) diff --git a/tn/hungarian/rules/telephone.py b/tn/hungarian/rules/telephone.py new file mode 100644 index 0000000..4e8ae7b --- /dev/null +++ b/tn/hungarian/rules/telephone.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import data_keys, data_map +from tn.processor import Processor + +TOKEN_ORDER = {"telephone": ["prompt", "country_code", "number_part"]} + + +class Telephone(Processor): + """Phone numbers and IPv4 addresses, e.g. ``+36-80/555-111``, ``06 40 49 4949``. + + Raw fields: an optional ``prompt`` (``tel.``), ``country_code`` and + ``number_part``; digit groups are read as numbers. + """ + + def __init__(self, cardinal): + super().__init__("telephone", token_orders=TOKEN_ORDER) + digit = cardinal.digit + two = cardinal.two_digits_read + three = cardinal.three_digits_read + four = cardinal.four_digits_read + up_to_three = (digit | two | three).optimize() + + separator = pynini.union(" ", pynini.cross("-", " ")) + area_separator = separator | pynini.cross("/", " ") + area = data_keys("telephone/area_codes.tsv") @ cardinal.plain + base = pynini.union( + three + separator + three, + three + separator + two + separator + two, + three + separator + four, + two + separator + four, + four + separator + three, + two + separator + two + separator + three, + ) + trunk = pynini.cross("06", "nulla hat") + trunk |= delete("(") + trunk + delete(")") + number = area + area_separator + base + ip = up_to_three + (pynini.cross(".", " pont ") + up_to_three) ** 3 + self.number = ((trunk + separator).ques + number | ip).optimize() + + plus = pynini.cross("+", "plusz ") | pynini.cross("00", "nulla nulla ") + country = data_keys("telephone/country_codes.tsv") @ cardinal.plain + country = delete("(").ques + country + delete(")").ques + self.country = (plus + delete(" ").ques + country).optimize() + self.prompt = data_map("telephone/telephone_abbr.tsv") + + number_t = self.tag_field("number_part", self.number) + country_t = self.tag_field("country_code", self.country) + delete(pynini.union(" ", "-")) + insert(" ") + prompt_t = self.tag_field("prompt", self.prompt) + delete(":").ques + pynini.accep(" ") + self.tagger = self.add_tokens(prompt_t.ques + country_t.ques + number_t) + + prompt_v = self.verbalize_field("prompt", self.prompt) + pynini.accep(" ") + country_v = self.verbalize_field("country_code", self.country) + pynini.accep(" ") + number_v = self.verbalize_field("number_part", self.number) + self.verbalizer = self.delete_tokens(prompt_v.ques + country_v.ques + number_v) diff --git a/tn/hungarian/rules/time.py b/tn/hungarian/rules/time.py new file mode 100644 index 0000000..828accf --- /dev/null +++ b/tn/hungarian/rules/time.py @@ -0,0 +1,106 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.hungarian.rules.utils import DIGIT, NONZERO, data_labels, inflect_abbreviation, naive_inflector +from tn.processor import Processor + +TOKEN_ORDER = {"time": ["hours", "minutes", "seconds", "suffix", "zone"]} + + +def unit_forms(abbr, word): + """A time unit with its abbreviation and inflected forms, e.g. ``p-cel`` -> ``perccel``.""" + + forms = pynini.string_map(naive_inflector(abbr, word, True) + [(abbr, word)]) + return (forms | pynini.project(forms, "output")).optimize() + + +class Time(Processor): + """Times, e.g. ``08:05`` -> ``nyolc óra öt perc``, ``8 ó 5 p-cel`` -> ``nyolc óra öt perccel``. + + Raw ``hours``/``minutes``/``seconds``; a trailing (inflected) unit is kept + in ``suffix`` and a time zone in ``zone``. ``00`` minutes or seconds are + kept raw and silenced by the verbalizer. + """ + + def __init__(self, cardinal): + super().__init__("time", token_orders=TOKEN_ORDER) + hour_forms = unit_forms("ó", "óra") + minute_forms = unit_forms("p", "perc") + second_forms = unit_forms("mp", "másodperc") + hour_word = pynini.union("ó", "óra") + minute_word = pynini.union("p", "perc") + + zone = pynini.string_map([(row[0], row[1]) for row in data_labels("time/time_zone.tsv")]) + for _, spelled in data_labels("time/time_zone.tsv"): + zone |= pynini.string_map([(form.replace(" ", ""), form) + for form, _ in inflect_abbreviation(spelled, "a", True)]) + self.zone = zone.optimize() + + def numbers(first, last): + return pynini.union(*[str(x) for x in range(first, last + 1)]) + + hour = numbers(0, 24) @ cardinal.plain + minute = numbers(1, 59) @ cardinal.plain + self.hour = ((delete("0").ques + DIGIT | NONZERO + DIGIT) @ hour).optimize() + self.minute = ((delete("0") + NONZERO | NONZERO + DIGIT) @ minute).optimize() + self.loose_minute = ((delete("0").ques + DIGIT | NONZERO + DIGIT) @ minute).optimize() + zero = pynini.accep("00") + + colon = delete(":") + space = insert(" ") + optional_space = delete(" ").ques + space + hours_t = self.tag_field("hours", self.hour) + minutes_t = self.tag_field("minutes", self.minute | zero) + seconds_t = self.tag_field("seconds", self.minute | zero) + loose_minutes_t = self.tag_field("minutes", self.loose_minute) + loose_seconds_t = self.tag_field("seconds", self.loose_minute) + zone_t = pynini.accep(" ") + self.tag_field("zone", self.zone) + + def suffix_t(forms): + return self.tag_field("suffix", forms) + + # 08:05, 08:05:05, optionally with a time zone. + clock = hours_t + colon + space + minutes_t + (colon + space + seconds_t).ques + zone_t.ques + # 8 ó-ban, 8 óra CET + spoken = hours_t + pynini.accep(" ") + (suffix_t(hour_forms) | suffix_t(hour_word) + zone_t) + # 8 ó 5 p-cel, 8 ó 5 perc CET + units = hours_t + optional_space + delete(hour_word + " ") + loose_minutes_t + optional_space + spoken |= units + (suffix_t(minute_forms) | suffix_t(minute_word) + zone_t) + # 8 ó 5 p 3 mp-kor + spoken |= (units + delete(minute_word + " ") + loose_seconds_t + optional_space + + suffix_t(second_forms)) + self.tagger = self.add_tokens(clock | spoken) + + hours_v = self.verbalize_field("hours", self.hour) + minute_value = self.minute | self.loose_minute + + def unit_v(name, word): + value = pynini.accep(" ") + self.verbalize_field(name, minute_value) + insert(" " + word) + return value | self.verbalize_field(name, delete(zero), leading_space=True) + + def value_v(name): + return pynini.accep(" ") + self.verbalize_field(name, minute_value) + + suffix_v = pynini.accep(" ") + self.verbalize_field("suffix", hour_forms | minute_forms | second_forms) + zone_v = (pynini.accep(" ") + self.verbalize_field("zone", self.zone)).ques + hour_v = hours_v + insert(" óra") + graph = hour_v + unit_v("minutes", "perc") + unit_v("seconds", "masodperc").ques + graph |= hours_v + suffix_v + graph |= hour_v + value_v("minutes") + suffix_v + graph |= hour_v + value_v("minutes") + insert(" perc") + value_v("seconds") + suffix_v + self.verbalizer = self.delete_tokens(graph + zone_v) diff --git a/tn/hungarian/rules/utils.py b/tn/hungarian/rules/utils.py new file mode 100644 index 0000000..2561663 --- /dev/null +++ b/tn/hungarian/rules/utils.py @@ -0,0 +1,156 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hungarian alphabet, data loading and the naive suffix inflector.""" + +from functools import lru_cache + +import pynini +from pynini.lib import byte, utf8 + +from tn.utils import get_abs_path, load_labels + +SIGMA = utf8.VALID_UTF8_CHAR.star +DIGIT = byte.DIGIT +NONZERO = pynini.difference(DIGIT, "0").optimize() +_ALPHA_UPPER = "AÁBCDEÉFGHIÍJKLMNOÓÖŐPQRSTUÚÜŰVWXYZ" +_ALPHA_LOWER = "aábcdeéfghiíjklmnoóöőpqrstuúüűvwxyz" +HU_ALPHA = pynini.union(*(_ALPHA_UPPER + _ALPHA_LOWER)).optimize() +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(_ALPHA_UPPER, _ALPHA_LOWER)]).optimize() +TO_UPPER = pynini.invert(TO_LOWER).optimize() + + +def data_path(name): + return get_abs_path("hungarian/data/" + name) + + +def data_labels(name): + return [row for row in load_labels(data_path(name)) if row] + + +def data_map(name, invert=False): + """Maps column 0 to column 1 (or the reverse); one-column rows map to themselves.""" + + pairs = [] + for row in data_labels(name): + key, value = row[0], row[1] if len(row) > 1 else row[0] + pairs.append((value, key) if invert else (key, value)) + return pynini.string_map(pairs).optimize() + + +def data_keys(name): + return pynini.union(*[pynini.escape(row[0]) for row in data_labels(name)]).optimize() + + +def suffix_rewrite(mapping, exceptions=None): + """Rewrites the end of a bounded word with ``mapping``. + + ``exceptions`` (whole-word replacements) take precedence; the result is + meant to be composed onto a bounded spoken graph. + """ + + graph = SIGMA + mapping + if exceptions is None: + return graph.optimize() + keys = pynini.project(exceptions, "input") + return (exceptions | pynini.difference(SIGMA, keys) @ graph).optimize() + + +@lru_cache(maxsize=None) +def _inflection_tables(): + def load(name): + return {row[0]: row[1].split(" ") for row in data_labels("inflection/" + name)} + + return load("endings.tsv"), load("plural_endings.tsv"), load("word_endings.tsv") + + +def _modify_ending(outword, word, form): + """Adjusts endings that are written differently after abbreviations.""" + + if outword == word: + return form + undouble = {"nny": "ny", "lly": "ly"} + for ending in ["ny", "nny", "ly", "lly", "ev", "év", "út", "ut", "a", "á", "e", "é"]: + if form.startswith(ending): + return undouble.get(ending, "") + form[len(ending):] + return form + + +def naive_inflector(abbr, word, singular_only=False): + """Inflects an abbreviation together with its expansion. + + Returns ``(inflected abbreviation, inflected expansion)`` pairs, e.g. + ``("tv.-t", "törvényt")``; possessive forms are omitted. + """ + + singular, plural, lexical = _inflection_tables() + keys_sorted = sorted(singular, key=len, reverse=True) + if word in lexical: + key, ends = word, lexical[word] + else: + key = next((key for key in keys_sorted if word.endswith(key)), None) + if key is None: + raise KeyError("no inflection class for {}".format(word)) + ends = singular[key] + outword = word + for wordend in ["ny", "ly", "év", "út", "a", "e"]: + if outword.endswith(wordend): + outword = outword[:-len(wordend)] + if "-" in abbr: + abbr = abbr.split("-")[0] + forms = [("{}-{}".format(abbr, _modify_ending(outword, word, form)), outword + form) for form in ends] + if not singular_only: + for plural_form in plural[key]: + plural_key = key + "k" if plural_form == "k" else plural_form + tweaked = _modify_ending(outword, word, plural_form) + forms.append(("{}-{}".format(abbr, tweaked), outword + plural_form)) + for form in singular[plural_key]: + forms.append(("{}-{}{}".format(abbr, tweaked, form), outword + plural_form + form)) + return forms + + +def inflect_abbreviation(abbr, word, singular_only=False): + """Inflects an abbreviation by the name of its last letter. + + Returns ``(inflected abbreviation, inflected expansion)`` pairs, e.g. + ``("GBP-t", "angol fontot")``. + """ + + last = abbr[-1].lower() + if last in "bcdgjptvz": + ending = "é" + elif last in "aáeéiíoóöőuúüű": + ending = last + elif last in "flmnrs": + ending = "e" + last + elif last in "hk": + ending = "á" + else: + return [] + word_part = [form[1] for form in naive_inflector(".", word, singular_only)] + abbr_part = [form[0] for form in naive_inflector(abbr, ending, singular_only)] + return list(zip(abbr_part, word_part)) + + +def inflected_map(name, singular_only=False, lower=False, skip_spaces=True): + """Loads an abbreviation table together with its inflected forms.""" + + forms = [] + for row in data_labels(name): + key = row[0].lower() if lower else row[0] + forms.append((key, row[1])) + if not (skip_spaces and " " in row[1]): + forms += naive_inflector(key, row[1], singular_only) + return pynini.string_map(forms).optimize() diff --git a/tn/hungarian/rules/whitelist.py b/tn/hungarian/rules/whitelist.py new file mode 100644 index 0000000..805718e --- /dev/null +++ b/tn/hungarian/rules/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tn.hungarian.rules.utils import data_map, inflected_map +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class WhiteList(Processor): + """Abbreviations with their case endings, e.g. ``tv.-nyel`` -> ``törvénnyel``.""" + + def __init__(self): + super().__init__("whitelist", token_orders=TOKEN_ORDER) + graph = data_map("whitelist.tsv") + graph |= inflected_map("whitelist_inflect.tsv") + graph |= inflected_map("whitelist_inflect_sg.tsv", singular_only=True) + self.graph = graph.optimize() + self.tagger = self.add_tokens(self.tag_field("value", self.graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.graph)) diff --git a/tn/hungarian/rules/word.py b/tn/hungarian/rules/word.py new file mode 100644 index 0000000..cda5d8c --- /dev/null +++ b/tn/hungarian/rules/word.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from tn.processor import Processor + +TOKEN_ORDER = {} + + +class Word(Processor): + """Any other non-space run is copied verbatim.""" + + def __init__(self, punctuation): + super().__init__("word", token_orders=TOKEN_ORDER) + whitespace = pynini.union(" ", "\t", "\n", "\r", "\u00a0") + # Exclude whole punctuation tokens, not punctuation embedded in words. + self.graph = ((self.VCHAR - whitespace).plus - punctuation.graph).optimize() + self.value_graph = ( + ((self.VCHAR - " ").plus - self.SPACE.plus) + @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", self.VSIGMA) + | pynutil.delete(self.SPACE.plus) + ).optimize() + self.tagger = self.add_tokens(self.tag_field("value", self.graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.value_graph)) diff --git a/tn/hungarian/test/__init__.py b/tn/hungarian/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tn/hungarian/test/data/test_cases_cardinal.txt b/tn/hungarian/test/data/test_cases_cardinal.txt new file mode 100644 index 0000000..865f8b2 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_cardinal.txt @@ -0,0 +1,69 @@ +2~kettő +3~három +4~négy +5~öt +6~hat +7~hét +8~nyolc +9~kilenc +10~tíz +11~tizenegy +12~tizenkettő +13~tizenhárom +14~tizennégy +15~tizenöt +16~tizenhat +17~tizenhét +18~tizennyolc +19~tizenkilenc +20~húsz +22~huszonkettő +23~huszonhárom +24~huszonnégy +25~huszonöt +26~huszonhat +27~huszonhét +28~huszonnyolc +29~huszonkilenc +30~harminc +40~negyven +50~ötven +60~hatvan +70~hetven +80~nyolcvan +90~kilencven +100~száz +102~százkettő +120~százhúsz +130~százharminc +1000~ezer +1 000~ezer +1.000~ezer +1010~ezertíz +1020~ezerhúsz +1100~ezerszáz +1110~ezerszáztíz +1111~ezerszáztizenegy +2000~kettőezer +2002~kettőezer-kettő +2010~kettőezer-tíz +2020~kettőezer-húsz +2100~kettőezer-száz +2110~kettőezer-száztíz +2111~kettőezer-száztizenegy +10000~tízezer +10 000~tízezer +10.000~tízezer +100000~százezer +100 000~százezer +100.000~százezer +1 000 000~millió +1.000.000~millió +1 034 068~millió-harmincnégyezer-hatvannyolc +2.000.000~kettőmillió +1.000.000.000~milliárd +2.000.000.000~kettőmilliárd +3 000 000 000 000~hárombillió +3.000.000.000.000~hárombillió +100 000 000 000 000 000 000 000~száztrilliárd +100 000 000 000 000 000 000 002~száztrilliárd-kettő diff --git a/tn/hungarian/test/data/test_cases_date.txt b/tn/hungarian/test/data/test_cases_date.txt new file mode 100644 index 0000000..bcb71ab --- /dev/null +++ b/tn/hungarian/test/data/test_cases_date.txt @@ -0,0 +1,17 @@ +2010. április 1.~kettőezer-tíz április elseje +2010. ápr. 1.~kettőezer-tíz április elseje +2010. IV. 1.~kettőezer-tíz április elseje +2010. 04. 1.~kettőezer-tíz április elseje +2010. 4. 1.~kettőezer-tíz április elseje +2010. 04. 1-je~kettőezer-tíz április elseje +2010. 04. 1-jén~kettőezer-tíz április elsején +2010. 04. 1-én~kettőezer-tíz április elsején +április 1.~április elseje +ápr. 1.~április elseje +IV. 1.~április elseje +04. 1.~április elseje +4. 1.~április elseje +04. 1-je~április elseje +04. 1-jén~április elsején +04. 1-én~április elsején +2010-04-01~kettőezer-tíz április elseje diff --git a/tn/hungarian/test/data/test_cases_decimal.txt b/tn/hungarian/test/data/test_cases_decimal.txt new file mode 100644 index 0000000..103f7e0 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_decimal.txt @@ -0,0 +1,6 @@ +-11,4006 milliárd~mínusz tizenegy egész négyezer-hat tízezred milliárd +-11,4006~mínusz tizenegy egész négyezer-hat tízezred +11,4006~tizenegy egész négyezer-hat tízezred +11,04006~tizenegy egész négyezer-hat százezred +1 milliárd~egy milliárd +1 mrd~egy milliárd diff --git a/tn/hungarian/test/data/test_cases_electronic.txt b/tn/hungarian/test/data/test_cases_electronic.txt new file mode 100644 index 0000000..3d9ea6d --- /dev/null +++ b/tn/hungarian/test/data/test_cases_electronic.txt @@ -0,0 +1,8 @@ +abc@def.hu~a b c kukac d e f pont hu +https://www.nvidia.com~h t t p s kettőspont perjel perjel w w w pont nvidia pont com +https://www.nvidia.com.~h t t p s kettőspont perjel perjel w w w pont nvidia pont com . +@jensen~kukac j e n s e n +www.nasa.gov.~w w w pont n a s a pont gov . +www.enveedya.com.hu.~w w w pont e n v e e d y a pont com pont hu . +@jensen.me~kukac j e n s e n pont m e +@wezyr1986~kukac w e z y r egy kilenc nyolc hat \ No newline at end of file diff --git a/tn/hungarian/test/data/test_cases_fraction.txt b/tn/hungarian/test/data/test_cases_fraction.txt new file mode 100644 index 0000000..7f916a1 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_fraction.txt @@ -0,0 +1,4 @@ +⅔~kettő harmad +2/3~kettő harmad +21/3443~huszonegy háromezer-négyszáznegyvenharmad +121/3443~százhuszonegy háromezer-négyszáznegyvenharmad diff --git a/tn/hungarian/test/data/test_cases_measure.txt b/tn/hungarian/test/data/test_cases_measure.txt new file mode 100644 index 0000000..a473823 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_measure.txt @@ -0,0 +1,6 @@ +1kg~egy kilogramm +12kg~tizenkettő kilogramm +-12kg~mínusz tizenkettő kilogramm +3mm~három milliméter +3mm-ben~három milliméterben +2,3 mm~kettő egész három tized milliméter diff --git a/tn/hungarian/test/data/test_cases_money.txt b/tn/hungarian/test/data/test_cases_money.txt new file mode 100644 index 0000000..834c150 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_money.txt @@ -0,0 +1,9 @@ +$1~egy dollár +$1,00~egy dollár +$2~kettő dollár +$2,00~kettő dollár +$1,01~egy dollár egy cent +$1 mrd~egy milliárd dollár +1 mrd GBP-t~egy milliárd angol fontot +1,32 GBP-t~egy egész harminckettő század angol fontot +2,32 GBP-t~kettő egész harminckettő század angol fontot diff --git a/tn/hungarian/test/data/test_cases_ordinal.txt b/tn/hungarian/test/data/test_cases_ordinal.txt new file mode 100644 index 0000000..8d704a9 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_ordinal.txt @@ -0,0 +1,5 @@ +2.~második +31.~harmincegyedik +455456.~négyszázötvenötezer-négyszázötvenhatodik +102002302.~százkettőmillió-kettőezer-háromszázkettedik +131 111.~százharmincegyezer-száztizenegyedik diff --git a/tn/hungarian/test/data/test_cases_telephone.txt b/tn/hungarian/test/data/test_cases_telephone.txt new file mode 100644 index 0000000..eb782c0 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_telephone.txt @@ -0,0 +1,9 @@ ++ (36) 1 441-4000~plusz harminchat egy négyszáznegyvenegy négyezer ++36 1 4888 700~plusz harminchat egy négyezer-nyolcszáznyolcvannyolc hétszáz +hívjon a +36 1 4888 700~hívjon a plusz harminchat egy négyezer-nyolcszáznyolcvannyolc hétszáz ++36-80/555-111~plusz harminchat nyolcvan ötszázötvenöt száztizenegy +tel.: +36-80/555-111~telefon plusz harminchat nyolcvan ötszázötvenöt száztizenegy +06 40 49 4949~nulla hat negyven negyvenkilenc négyezer-kilencszáznegyvenkilenc +06-40-49-4949~nulla hat negyven negyvenkilenc négyezer-kilencszáznegyvenkilenc +tel. 06-40-49-4949~telefon nulla hat negyven negyvenkilenc négyezer-kilencszáznegyvenkilenc +Az IP-címed 193.1.89.60~Az IP-címed százkilencvenhárom pont egy pont nyolcvankilenc pont hatvan diff --git a/tn/hungarian/test/data/test_cases_time.txt b/tn/hungarian/test/data/test_cases_time.txt new file mode 100644 index 0000000..f0f7814 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_time.txt @@ -0,0 +1,10 @@ +08:00~nyolc óra +08:05~nyolc óra öt perc +8 ó-ban~nyolc órában +8 ó 5 p-cel~nyolc óra öt perccel +08:00:05~nyolc óra öt masodperc +08:05:00~nyolc óra öt perc +08:05:05~nyolc óra öt perc öt masodperc +00:00~nulla óra +00:00 cet~nulla óra c e t +00:00 cet-t~nulla óra c e t-t diff --git a/tn/hungarian/test/data/test_cases_whitelist.txt b/tn/hungarian/test/data/test_cases_whitelist.txt new file mode 100644 index 0000000..7bfd7d4 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_whitelist.txt @@ -0,0 +1,10 @@ +tv.-t~törvényt +tv.-nyel~törvénnyel +tv.-ekként~törvényekként +fn.-et~főnevet +fn.-eket~főneveket +fn.-vel~főnévvel +isk.-t~iskolát +isk.-kat~iskolákat +isk.-val~iskolával +kir.-ok~királyok diff --git a/tn/hungarian/test/data/test_cases_word.txt b/tn/hungarian/test/data/test_cases_word.txt new file mode 100644 index 0000000..481e674 --- /dev/null +++ b/tn/hungarian/test/data/test_cases_word.txt @@ -0,0 +1,52 @@ +~ +yahoo!~yahoo! +veinte!~veinte! +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang +tőled~tőled +köszönöm~köszönöm +találni~találni +szép~szép diff --git a/tn/hungarian/test/normalizer_test.py b/tn/hungarian/test/normalizer_test.py new file mode 100644 index 0000000..6a81725 --- /dev/null +++ b/tn/hungarian/test/normalizer_test.py @@ -0,0 +1,210 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from tn.hungarian.normalizer import Normalizer + + +def original_cases(): + cases = [] + for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for index, line in enumerate(stream, 1): + if line.endswith("\n"): + line = line[:-1] + written, spoken = line.split("~") + cases.append(pytest.param(written, spoken, id="{}-{}".format(path.stem, index))) + return cases + + +CASES = original_cases() + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(Normalizer) + + +def test_original_fixture_count(): + # 204 newline-terminated lines plus the unterminated last electronic case. + assert len(CASES) == 205 + assert CASES[0].values == ("2", "kettő") + assert CASES[153].values == ("", "") + + +@pytest.mark.parametrize("written, spoken", CASES) +def test_original(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", CASES) +def test_original_mapping(normalizer, written, spoken): + result = normalizer.normalize_with_mapping(written, include_identity=True) + assert result.output_text == spoken + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert spoken[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + written, spoken = param.values + assert normalizer.verbalize(normalizer.tag(written)) == spoken + + +@pytest.mark.parametrize("written, kind, members, spoken", [ + ("1 000", "cardinal", {"integer": "1 000"}, "ezer"), + ("-12", "cardinal", {"negative": "-", "integer": "12"}, "mínusz tizenkettő"), + ("131 111.", "ordinal", {"integer": "131 111"}, "százharmincegyezer-száztizenegyedik"), + ("-11,4006 milliárd", "decimal", { + "negative": "-", "integer_part": "11", "fractional_part": "4006", "quantity": "milliárd", + }, "mínusz tizenegy egész négyezer-hat tízezred milliárd"), + ("1 mrd", "decimal", {"integer_part": "1", "quantity": "mrd"}, "egy milliárd"), + ("21/3443", "fraction", {"numerator": "21", "denominator": "3443"}, + "huszonegy háromezer-négyszáznegyvenharmad"), + ("⅔", "fraction", {"symbol": "⅔"}, "kettő harmad"), + ("1 1/2", "fraction", {"integer_part": "1", "numerator": "1", "denominator": "2"}, "egy és egy fél"), + ("3mm-ben", "measure", {"integer": "3", "units": "mm-ben"}, "három milliméterben"), + ("2,3 mm", "measure", {"integer_part": "2", "fractional_part": "3", "units": "mm"}, + "kettő egész három tized milliméter"), + ("$1,01", "money", {"currency": "$", "integer_part": "1", "minor": "01"}, "egy dollár egy cent"), + ("$0,50", "money", {"currency": "$", "integer_part": "0", "minor": "50"}, "ötven cent"), + ("$2,00", "money", {"currency": "$", "integer_part": "2", "fractional_part": "00"}, "kettő dollár"), + ("1,32 GBP-t", "money", {"integer_part": "1", "fractional_part": "32", "currency": "GBP-t"}, + "egy egész harminckettő század angol fontot"), + ("1 mrd Ft", "money", {"integer_part": "1", "quantity": "mrd", "currency": "Ft"}, "egy milliárd forint"), + ("2010. ápr. 1-jén", "date", {"year": "2010", "month": "ápr", "day": "1-jén"}, + "kettőezer-tíz április elsején"), + ("2010-04-01", "date", {"year": "2010", "month": "04", "day": "01"}, "kettőezer-tíz április elseje"), + ("1.4.2010", "date", {"day": "1", "month": "4", "year": "2010"}, "kettőezer-tíz április elseje"), + ("08:05:00", "time", {"hours": "08", "minutes": "05", "seconds": "00"}, "nyolc óra öt perc"), + ("8 ó 5 p-cel", "time", {"hours": "8", "minutes": "5", "suffix": "p-cel"}, "nyolc óra öt perccel"), + ("00:00 cet-t", "time", {"hours": "00", "minutes": "00", "zone": "cet-t"}, "nulla óra c e t-t"), + ("tel.: +36-80/555-111", "telephone", {"prompt": "tel.", "country_code": "+36", "number_part": "80/555-111"}, + "telefon plusz harminchat nyolcvan ötszázötvenöt száztizenegy"), + ("193.1.89.60", "telephone", {"number_part": "193.1.89.60"}, + "százkilencvenhárom pont egy pont nyolcvankilenc pont hatvan"), + ("abc@def.hu", "electronic", {"username": "abc", "domain": "def.hu"}, "a b c kukac d e f pont hu"), + ("https://www.nvidia.com", "electronic", {"protocol": "https://www.", "domain": "nvidia.com"}, + "h t t p s kettőspont perjel perjel w w w pont nvidia pont com"), + ("@wezyr1986", "electronic", {"username": "wezyr1986"}, "kukac w e z y r egy kilenc nyolc hat"), + ("tv.-nyel", "whitelist", {"value": "tv.-nyel"}, "törvénnyel"), + ("és tsai.", "whitelist", {"value": "és tsai."}, "és társai"), + ("köszönöm", "word", {"value": "köszönöm"}, "köszönöm"), +]) +def test_raw_fields(normalizer, written, kind, members, spoken): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == spoken + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", [ + ("(5)", "(öt)"), + ("5,", "öt,"), + ("100-200", "száz-kétszáz"), + ("100 - 200", "száz - kétszáz"), + ("1 034 068", "millió-harmincnégyezer-hatvannyolc"), + ("Április 5.", "április ötödike"), + ("2010. május", "kettőezer-tíz május"), + ("8 óra cet", "nyolc óra c e t"), + ("8 ó 5 p 3 mp", "nyolc óra öt perc három másodperc"), + ("5%", "öt százalék"), + ("$1,5", "egy dollár ötven cent"), + ("$1,234", "egy egész kétszázharmincnégy ezred dollár"), + ("2 USD-t", "kettő amerikai dollárt"), + ("0036 1 441 4000", "nulla nulla harminchat egy négyszáznegyvenegy négyezer"), + ("x@y.com", "x kukac y pont com"), + ("Bp.-en", "Budapesten"), +]) +def test_additional_semantics(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + assert normalizer.verbalize(normalizer.tag(written)) == spoken + assert normalizer.normalize_with_mapping(written, include_identity=True).output_text == spoken + + +@pytest.mark.parametrize("written, spoken, fields", [ + ("(5)", "(öt)", [("punctuation", {"value": "("}), ("cardinal", {"integer": "5"}), + ("punctuation", {"value": ")"})]), + ("5,", "öt,", [("cardinal", {"integer": "5"}), ("punctuation", {"value": ","})]), + ("5 !", "öt !", [("cardinal", {"integer": "5"}), ("punctuation", {"value": " !"})]), + ("( 5 )", "( öt )", [("punctuation", {"value": "( "}), ("cardinal", {"integer": "5"}), + ("punctuation", {"value": " )"})]), +]) +def test_punctuation_spacing(normalizer, written, spoken, fields): + assert normalizer.normalize(written) == spoken + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert [(token.name, token.members) for token in parser.tokens] == fields + assert normalizer.verbalize(tagged) == spoken + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert outputs[0] == spoken + assert len(outputs) == len(set(outputs)) + assert [result.output_text for result in mapped] == outputs + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +@pytest.mark.parametrize("written", ["1/2", "15:52", "$1,01", "131 111.", "2010. 04. 1-jén", "tv.-nyel"]) +def test_nbest_mapping_synchronization(normalizer, written): + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(written) + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + written = 'út\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(written)) + assert [token.members for token in parser.tokens] == [{"value": written}] + assert normalizer.normalize(written) == written + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("written", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, written): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": written} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(written, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) diff --git a/tn/test/cli_test.py b/tn/test/cli_test.py index 514677b..6f03356 100644 --- a/tn/test/cli_test.py +++ b/tn/test/cli_test.py @@ -60,6 +60,7 @@ def verbalize(self, tagged): ("tn", "sv", "tn.swedish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "es", "tn.spanish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "hi", "tn.hindi.normalizer", "Normalizer", [], {}), + ("tn", "hu", "tn.hungarian.normalizer", "Normalizer", [], {}), ("tn", "he", "tn.hebrew.normalizer", "Normalizer", [], {}), ("tn", "it", "tn.italian.normalizer", "Normalizer", [], {}), ("tn", "mr", "tn.marathi.normalizer", "Normalizer", [], {}), @@ -98,6 +99,7 @@ def verbalize(self, tagged): ("itn", "sv", "itn.swedish.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "es", "itn.spanish.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "hi", "itn.hindi.inverse_normalizer", "InverseNormalizer", [], {}), + ("itn", "hu", "itn.hungarian.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "he", "itn.hebrew.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "it", "itn.italian.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "hi_en", "itn.hindi_english.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), @@ -232,6 +234,8 @@ def test_itn_only_languages_are_rejected_for_tn(language): ("itn", "es", "--cased", "--enable-million"), ("tn", "hi", "--cache-dir", "--deterministic"), ("itn", "hi", "--cache-dir", "--enable-million"), + ("tn", "hu", "--cache-dir", "--deterministic"), + ("itn", "hu", "--cache-dir", "--enable-million"), ("tn", "he", "--cache-dir", "--deterministic"), ("itn", "he", "--cache-dir", "--enable-million"), ("tn", "it", "--cache-dir", "--deterministic"),