AlaBoussoffara commited on
Commit
2491489
·
1 Parent(s): bc8f051

modified inference preprocessing

Browse files
Files changed (1) hide show
  1. src/mini_transformer/inference.py +18 -1
src/mini_transformer/inference.py CHANGED
@@ -115,6 +115,23 @@ def _get_or_create_session(
115
  return transformer, tokenizer
116
 
117
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  def run_inference(cfg: DictConfig) -> list[str]:
119
  """Run inference using a composed Hydra configuration."""
120
 
@@ -151,7 +168,7 @@ def run_inference(cfg: DictConfig) -> list[str]:
151
  device=device,
152
  )
153
 
154
- text_input = scfg.input_text
155
  if not text_input:
156
  raise SystemExit('Pass text like: input_text="hello world"')
157
 
 
115
  return transformer, tokenizer
116
 
117
 
118
+ def _preprocess_text(text: str) -> str:
119
+ """Lowercase and ensure sentence ends with punctuation.
120
+
121
+ Simple preprocessing used before tokenization during inference:
122
+ - Lowercase the entire input.
123
+ - If the last non-space character is not one of '.!?', append a period.
124
+ """
125
+ s = text.strip()
126
+ if not s:
127
+ return s
128
+ s = s.lower()
129
+ last = s[-1]
130
+ if last not in ".!?":
131
+ s += "."
132
+ return s
133
+
134
+
135
  def run_inference(cfg: DictConfig) -> list[str]:
136
  """Run inference using a composed Hydra configuration."""
137
 
 
168
  device=device,
169
  )
170
 
171
+ text_input = _preprocess_text(scfg.input_text)
172
  if not text_input:
173
  raise SystemExit('Pass text like: input_text="hello world"')
174