Skip to content

Hi TN: fraction class improvements with lexical mappings for special cases #420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mgrafu merged 6 commits into from
May 13, 2026
Merged

Hi TN: fraction class improvements with lexical mappings for special cases #420

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ca01229
Fix: fraction class improvements with lexical mappings
shrpawar-alt Apr 28, 2026
2761cfb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 28, 2026
35510f7
Merge branch 'staging/hi_tn_v3' into hi-tn-fraction
shrpawar-alt Apr 29, 2026
c8411c2
move common fraction mappings to TSV and add ASCII digit support
shrpawar-alt May 13, 2026
7e38699
Jenkins date conflict resolved
shrpawar-alt May 13, 2026
70c3ea1
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ pipeline {
MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-23-26-0'
HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-23-26-0'
HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/05-13-26-0'
DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
}
stages {
Expand Down
10 changes: 10 additions & 0 deletions nemo_text_processing/text_normalization/hi/data/fraction/common_fractions.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
१/२ आधा
१/३ तिहाई
२/३ दो तिहाई
१/४ चौथाई
३/४ तीन चौथाई
1/2 आधा
1/3 तिहाई
2/3 दो तिहाई
1/4 चौथाई
3/4 तीन चौथाई
25 changes: 19 additions & 6 deletions nemo_text_processing/text_normalization/hi/taggers/fraction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,18 @@
)
from nemo_text_processing.text_normalization.hi.utils import get_abs_path

HI_ONE_HALF = "१/२" # 1/2
HI_ONE_QUARTER = "१/४" # 1/4
HI_THREE_QUARTERS = "३/४" # 3/4
HI_ONE_HALF = "१/२"
HI_ONE_QUARTER = "१/४"
HI_THREE_QUARTERS = "३/४"


class FractionFst(GraphFst):
"""
Comment thread
mgrafu marked this conversation as resolved.
Finite state transducer for classifying fraction
"२३ ४/६" ->
fraction { integer: "तेईस" numerator: "चार" denominator: "छः"}
fraction { integer: "तेईस" numerator: "चार" denominator: "छह"}
४/६" ->
fraction { numerator: "चार" denominator: "छः"}
fraction { numerator: "चार" denominator: "छह"}


Args:
Expand All @@ -54,13 +54,16 @@ def __init__(self, cardinal, deterministic: bool = True):
self.optional_graph_negative = pynini.closure(
pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + pynutil.insert(NEMO_SPACE), 0, 1
)

self.integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"")

self.numerator = (
pynutil.insert("numerator: \"")
+ cardinal_graph
+ pynini.cross(pynini.union("/", NEMO_SPACE + "/" + NEMO_SPACE), "\"")
+ pynutil.insert(NEMO_SPACE)
)

self.denominator = pynutil.insert("denominator: \"") + cardinal_graph + pynutil.insert("\"")

dedh_dhai_graph = pynini.string_map(
Expand All @@ -77,6 +80,15 @@ def __init__(self, cardinal, deterministic: bool = True):
paune_numbers = paune + pynini.cross(NEMO_SPACE + HI_THREE_QUARTERS, "")
paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers

common_fraction_map = pynini.string_file(get_abs_path("data/fraction/common_fractions.tsv"))

graph_common_fraction = (
pynutil.insert("morphosyntactic_features: \"")
+ common_fraction_map
+ pynutil.insert("\"")
+ pynutil.insert(NEMO_SPACE)
)

graph_dedh_dhai = (
pynutil.insert("morphosyntactic_features: \"")
+ dedh_dhai_graph
Expand Down Expand Up @@ -114,10 +126,11 @@ def __init__(self, cardinal, deterministic: bool = True):

weighted_graph = (
final_graph
| pynutil.add_weight(graph_common_fraction, -0.3)
| pynutil.add_weight(graph_dedh_dhai, -0.2)
| pynutil.add_weight(graph_paune, -0.2)
| pynutil.add_weight(graph_savva, -0.1)
| pynutil.add_weight(graph_sadhe, -0.1)
| pynutil.add_weight(graph_paune, -0.2)
)

self.graph = weighted_graph
Expand Down
4 changes: 2 additions & 2 deletions nemo_text_processing/text_normalization/hi/verbalizers/fraction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
class FractionFst(GraphFst):
"""
Finite state transducer for verbalizing fraction
e.g. fraction { integer: "तेईस" numerator: "चार" denominator: "छः" }-> तेईस चार बटा छः
e.g. fraction { numerator: "चार" denominator: "छः" } -> चार बटा छः
e.g. fraction { integer: "तेईस" numerator: "चार" denominator: "छह" }-> तेईस और चार बटा छह
e.g. fraction { numerator: "चार" denominator: "छह" } -> चार बटा छह
Args:
Expand Down
6 changes: 5 additions & 1 deletion tests/nemo_text_processing/hi/data_text_normalization/test_cases_fraction.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,8 @@
१०००००००००००००००/८~एक पद्म बटा आठ
100000000000000000/412~एक शंख बटा चार सौ बारह
२ २/७~दो और दो बटा सात
120 75/90~एक सौ बीस और पचहत्तर बटा नब्बे
120 75/90~एक सौ बीस और पचहत्तर बटा नब्बे
१/२~आधा
१/३~तिहाई
1/4~चौथाई
3/4~तीन चौथाई