entropy25 commited on
Commit
4d25cd1
·
verified ·
1 Parent(s): 43d8748

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +157 -403
app.py CHANGED
@@ -5,16 +5,15 @@ from peft import PeftModel
5
  from functools import lru_cache
6
  import os
7
  import time
 
8
  import nltk
9
- import json
10
- import re
11
 
12
  try:
13
- nltk.data.find('tokenizers/punkt')
14
  except LookupError:
15
- nltk.download('punkt')
16
  try:
17
- nltk.download('punkt_tab')
18
  except:
19
  pass
20
 
@@ -24,332 +23,164 @@ BASE_MODEL = os.getenv("BASE_MODEL", "facebook/nllb-200-distilled-600M")
24
  ADAPTER_EN_TO_NO = os.getenv("ADAPTER_EN_TO_NO", "entropy25/mt_en_no_oil")
25
  ADAPTER_NO_TO_EN = os.getenv("ADAPTER_NO_TO_EN", "entropy25/mt_no_en_oil")
26
 
27
- tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 
 
 
 
28
 
29
- print("Loading model with 8-bit quantization...")
30
- quantization_config = BitsAndBytesConfig(load_in_8bit=True)
31
 
32
  base_model = AutoModelForSeq2SeqLM.from_pretrained(
33
  BASE_MODEL,
34
- quantization_config=quantization_config,
35
- low_cpu_mem_usage=True
36
  )
37
 
38
- print("Loading adapters...")
39
  model = PeftModel.from_pretrained(base_model, ADAPTER_EN_TO_NO, adapter_name="en_to_no")
40
  model.load_adapter(ADAPTER_NO_TO_EN, adapter_name="no_to_en")
41
  model.eval()
42
 
43
- print("Loading terminology glossary...")
44
- try:
45
- with open('glossary.json', 'r', encoding='utf-8') as f:
46
- glossary_data = json.load(f)
47
-
48
- TERMINOLOGY_EN_TO_NO = {}
49
- TERMINOLOGY_NO_TO_EN = {}
50
-
51
- for entry in glossary_data:
52
- en_term = entry['en'].strip()
53
- no_term = entry['no'].strip()
54
- TERMINOLOGY_EN_TO_NO[en_term.lower()] = no_term
55
- TERMINOLOGY_NO_TO_EN[no_term.lower()] = en_term
56
-
57
- print(f"Loaded {len(TERMINOLOGY_EN_TO_NO)} terminology entries")
58
-
59
- except Exception as e:
60
- print(f"Warning: Could not load glossary.json: {e}")
61
- TERMINOLOGY_EN_TO_NO = {}
62
- TERMINOLOGY_NO_TO_EN = {}
63
-
64
- COMPILED_PATTERNS_EN_TO_NO = {
65
- term: re.compile(r'\b' + re.escape(term) + r'\b', re.IGNORECASE)
66
- for term in TERMINOLOGY_EN_TO_NO.keys()
67
- }
68
 
69
- COMPILED_PATTERNS_NO_TO_EN = {
70
- term: re.compile(r'\b' + re.escape(term) + r'\b', re.IGNORECASE)
71
- for term in TERMINOLOGY_NO_TO_EN.keys()
72
  }
73
 
74
- COMMON_ERRORS = {
75
- "en_to_no": {
76
- "mud weight": ["mudgevekten", "mudvekt", "slam vekt"],
77
- "christmas tree": ["juletræet", "jule tre", "juletre"],
78
- "permeability": ["permeabiliteten"],
79
- "porosity": ["porøsiteten"],
80
- "training": ["utdanning"],
81
- "working pressure": ["arbeidstrykk"],
82
- },
83
- "no_to_en": {
84
- "slamvekt": ["slam weight", "mudweight"],
85
- "juletre": ["yule tree", "christmas-tree"],
86
- "permeabilitet": ["permeabiliteten"],
87
- "porøsitet": ["porøsiteten"],
88
- }
89
  }
90
 
91
- COMPILED_ERRORS_EN_TO_NO = {}
92
- for source_term, error_variants in COMMON_ERRORS["en_to_no"].items():
93
- COMPILED_ERRORS_EN_TO_NO[source_term] = [
94
- re.compile(r'\b' + re.escape(variant) + r'\b', re.IGNORECASE)
95
- for variant in error_variants
96
- ]
97
-
98
- COMPILED_ERRORS_NO_TO_EN = {}
99
- for source_term, error_variants in COMMON_ERRORS["no_to_en"].items():
100
- COMPILED_ERRORS_NO_TO_EN[source_term] = [
101
- re.compile(r'\b' + re.escape(variant) + r'\b', re.IGNORECASE)
102
- for variant in error_variants
103
- ]
104
 
105
- MAX_FILE_SIZE = 1024 * 1024
106
- MAX_TEXT_LENGTH = 10000
107
- BATCH_SIZE = 10
108
- NUM_BEAMS = 3
109
- MAX_LENGTH = 256
110
 
111
  @lru_cache(maxsize=512)
112
  def cached_sent_tokenize(text):
113
  return tuple(sent_tokenize(text))
114
 
115
- def fix_number_format(text, target_lang):
116
- if target_lang == "Norwegian":
117
- text = re.sub(r'(\d),(\d{3})', r'\1 \2', text)
118
- text = re.sub(r'(\d)\.(\d{3})(?!\d)', r'\1 \2', text)
119
- text = re.sub(r'(\d)\.(\d{1,2})(?=\s|$|[^\d])', r'\1,\2', text)
120
- else:
121
- text = re.sub(r'(\d)\s(\d{3})', r'\1,\2', text)
122
- text = re.sub(r'(\d),(\d{1,2})(?=\s|$|[^\d])', r'\1.\2', text)
123
- return text
124
-
125
- def find_source_terms_in_input(text, direction):
126
- if direction == "en_to_no":
127
- term_dict = TERMINOLOGY_EN_TO_NO
128
- else:
129
- term_dict = TERMINOLOGY_NO_TO_EN
130
-
131
- if not term_dict:
132
- return []
133
-
134
- found_terms = []
135
- text_lower = text.lower()
136
-
137
- for source_term in sorted(term_dict.keys(), key=len, reverse=True):
138
- if source_term in text_lower:
139
- target_term = term_dict[source_term]
140
- found_terms.append((source_term, target_term))
141
-
142
- return found_terms
143
-
144
- def post_process_terminology(text, direction, found_terms, use_terminology):
145
- if not use_terminology or not text:
146
- return text
147
-
148
- if direction == "en_to_no":
149
- compiled_patterns = COMPILED_PATTERNS_EN_TO_NO
150
- compiled_errors = COMPILED_ERRORS_EN_TO_NO
151
- else:
152
- compiled_patterns = COMPILED_PATTERNS_NO_TO_EN
153
- compiled_errors = COMPILED_ERRORS_NO_TO_EN
154
-
155
- result = text
156
-
157
- for source_term, target_term in found_terms:
158
- def preserve_case(match):
159
- original = match.group(0)
160
- if original and original[0].isupper():
161
- return target_term.capitalize()
162
- return target_term.lower()
163
-
164
- if source_term in compiled_patterns:
165
- result = compiled_patterns[source_term].sub(preserve_case, result)
166
-
167
- if source_term in compiled_errors:
168
- for error_pattern in compiled_errors[source_term]:
169
- result = error_pattern.sub(preserve_case, result)
170
-
171
- result = fix_number_format(result, "Norwegian" if direction == "en_to_no" else "English")
172
-
173
- return result
174
-
175
- def highlight_terminology(text, found_terms):
176
- if not found_terms:
177
- return text
178
-
179
- highlighted = text
180
- for source_term, target_term in found_terms:
181
- pattern = re.compile(r'\b(' + re.escape(target_term) + r')\b', re.IGNORECASE)
182
- highlighted = pattern.sub(r'<mark style="background-color: #fff3cd; padding: 2px 4px; border-radius: 3px;">\1</mark>', highlighted)
183
-
184
- return highlighted
185
-
186
- def translate_core(text, source_lang, target_lang, use_terminology=True):
187
- if not text.strip() or source_lang == target_lang:
188
- return text, 0.0, []
189
-
190
- start_time = time.time()
191
-
192
- if source_lang == "English" and target_lang == "Norwegian":
193
- model.set_adapter("en_to_no")
194
- tgt_code = "nob_Latn"
195
- direction = "en_to_no"
196
- elif source_lang == "Norwegian" and target_lang == "English":
197
- model.set_adapter("no_to_en")
198
- tgt_code = "eng_Latn"
199
- direction = "no_to_en"
200
- else:
201
- return "Unsupported language pair", 0.0, []
202
-
203
- found_terms = find_source_terms_in_input(text, direction)
204
-
205
- original_paragraphs = text.split('\n')
206
- final_translated_paragraphs = []
207
-
208
- for paragraph in original_paragraphs:
209
  if not paragraph.strip():
210
- final_translated_paragraphs.append("")
211
  continue
212
-
213
  sentences = cached_sent_tokenize(paragraph)
214
- paragraph_results = []
215
-
216
  for i in range(0, len(sentences), BATCH_SIZE):
217
- batch = sentences[i:i+BATCH_SIZE]
218
-
219
  inputs = tokenizer(
220
  batch,
221
  return_tensors="pt",
222
  padding=True,
223
  truncation=True,
224
- max_length=MAX_LENGTH
225
  )
226
-
227
- if hasattr(model, 'device'):
228
- inputs = {k: v.to(model.device) for k, v in inputs.items()}
229
-
230
- with torch.inference_mode():
231
- outputs = model.generate(
232
- **inputs,
233
- forced_bos_token_id=tokenizer.convert_tokens_to_ids(tgt_code),
234
- max_length=MAX_LENGTH,
235
- num_beams=NUM_BEAMS,
236
- early_stopping=True
237
- )
238
-
239
- batch_texts = tokenizer.batch_decode(outputs, skip_special_tokens=True)
240
- paragraph_results.extend(batch_texts)
241
-
242
- final_translated_paragraphs.append(" ".join(paragraph_results))
243
-
244
- raw_translation = '\n'.join(final_translated_paragraphs)
245
- corrected_translation = post_process_terminology(raw_translation, direction, found_terms, use_terminology)
246
- elapsed_time = time.time() - start_time
247
-
248
- return corrected_translation, elapsed_time, found_terms
249
 
250
- @lru_cache(maxsize=512)
251
- def translate_cached(text, source_lang, target_lang, use_terminology):
252
- result, elapsed, terms = translate_core(text, source_lang, target_lang, use_terminology)
253
- return result, elapsed, len(terms)
254
 
255
- def translate(text, source_lang, target_lang, use_terminology):
 
 
 
 
 
256
  try:
257
- if len(text) > MAX_TEXT_LENGTH:
258
- return f"Error: Text too long (max {MAX_TEXT_LENGTH:,} characters)", "", ""
259
-
260
- if not text.strip():
261
- return "", "", ""
262
-
263
- result, elapsed, terms_count = translate_cached(text, source_lang, target_lang, use_terminology)
264
-
265
- terminology_status = f"{terms_count} terms enforced" if use_terminology and terms_count > 0 else "No terminology enforcement" if not use_terminology else "No terms found"
266
- time_info = f"Completed in {elapsed:.2f}s | {terminology_status}"
267
-
268
- found_terms = find_source_terms_in_input(text, "en_to_no" if source_lang == "English" else "no_to_en")
269
- highlighted_result = highlight_terminology(result, found_terms) if use_terminology else result
270
-
271
- return result, highlighted_result, time_info
272
-
273
  except Exception as e:
274
- return f"Translation error: {str(e)}", "", ""
 
275
 
276
  def swap_languages(src, tgt, input_txt, output_txt):
277
  return tgt, src, output_txt, input_txt
278
 
 
279
  def load_file(file):
280
  if file is None:
281
- return "", "", ""
282
-
283
  try:
284
- if os.path.getsize(file.name) > MAX_FILE_SIZE:
285
- return "Error: File too large (max 1MB)", "", ""
286
-
287
- with open(file.name, 'r', encoding='utf-8') as f:
288
- content = f.read()
289
- if len(content) > MAX_TEXT_LENGTH:
290
- return f"Error: File content too long (max {MAX_TEXT_LENGTH:,} characters)", "", ""
291
- return content, "", ""
292
- except:
293
- try:
294
- with open(file.name, 'r', encoding='latin-1') as f:
295
- content = f.read()
296
  if len(content) > MAX_TEXT_LENGTH:
297
- return f"Error: File content too long (max {MAX_TEXT_LENGTH:,} characters)", "", ""
298
- return content, "", ""
299
- except Exception as e:
300
- return f"Error reading file: {str(e)}", "", ""
301
-
302
- EXAMPLES_EN = {
303
- "drilling_short": "Mud weight adjusted to 1.82 specific gravity at 3,247 meters depth.",
304
- "drilling_long": "The drilling operation at well site A-15 encountered unexpected high-pressure zones at 3,247 meters depth, requiring immediate adjustment of mud weight from 1.65 to 1.82 specific gravity to maintain wellbore stability and prevent potential kicks.",
305
- "reservoir_short": "Permeability is 250 millidarcy with 22 percent porosity.",
306
- "reservoir_long": "The reservoir shows excellent permeability of 250 millidarcy and porosity of 22 percent based on core analysis, indicating significant hydrocarbon potential with estimated oil saturation of 65 percent.",
307
- "subsea_short": "Christmas tree rated for 10,000 psi working pressure.",
308
- "subsea_long": "The subsea production system consists of a vertical Christmas tree rated for 10,000 psi working pressure and 150 degrees Celsius temperature, equipped with redundant safety features including automatic shutdown valves and real-time pressure monitoring systems.",
309
- "seismic_short": "Structural trap area estimated at 12 square kilometers.",
310
- "seismic_long": "Seismic data confirms the presence of a structural trap with an estimated area of 12 square kilometers, and productivity tests show stabilized oil production of 3,400 barrels per day at optimization pressure of 2,100 psi.",
311
- "safety_short": "H2S training required before site access.",
312
- "safety_long": "Emergency response procedures require all personnel to complete H2S safety training before site access, with breathing apparatus and wind indicators positioned at designated muster points, and immediate evacuation protocols activated when gas detection exceeds 10 ppm concentration levels."
313
- }
314
-
315
- EXAMPLES_NO = {
316
- "drilling_short": "Slamvekt justert til 1,82 spesifikk tyngde ved 3 247 meters dybde.",
317
- "drilling_long": "Boreoperasjonen ved brønnsted A-15 støtte på uventede høytrykksoner ved 3 247 meters dybde, noe som krevde umiddelbar justering av slamvekt fra 1,65 til 1,82 spesifikk tyngde for å opprettholde brønnborestabilitet og forhindre potensielle kicks.",
318
- "reservoir_short": "Permeabilitet er 250 millidarcy med 22 prosent porøsitet.",
319
- "reservoir_long": "Reservoaret viser utmerket permeabilitet på 250 millidarcy og porøsitet på 22 prosent basert på kjerneanalyse, noe som indikerer betydelig hydrokarbonpotensial med estimert oljemetning på 65 prosent.",
320
- "subsea_short": "Juletre dimensjonert for 10 000 psi arbeidstrykk.",
321
- "subsea_long": "Subsea produksjonssystemet består av et vertikalt juletre dimensjonert for 10 000 psi arbeidstrykk og 150 grader Celsius temperatur, utstyrt med redundante sikkerhetsfunksjoner inkludert automatiske nedstengningsventiler og sanntids trykkmonitorering.",
322
- "seismic_short": "Strukturell felle estimert til 12 kvadratkilometer.",
323
- "seismic_long": "Seismiske data bekrefter tilstedeværelsen av en strukturell felle med estimert areal på 12 kvadratkilometer, og produktivitetstester viser stabilisert oljeproduksjon på 3 400 fat per dag ved optimaliseringstrykk på 2 100 psi.",
324
- "safety_short": "H2S-opplæring påkrevd før tilgang til området.",
325
- "safety_long": "Nødprosedyrer krever at alt personell fullfører H2S-sikkerhetsopplæring før områdetilgang, med åndedrettsutstyr og vindindikatorer plassert ved utpekte samlingspunkter, og umiddelbare evakueringsprotokoller aktiveres når gassdeteksjon overskrider 10 ppm konsentrasjonsnivå."
326
- }
327
-
328
- def get_examples(source_lang):
329
- return EXAMPLES_EN if source_lang == "English" else EXAMPLES_NO
330
 
331
- def update_example_buttons(source_lang):
332
- examples = get_examples(source_lang)
333
- return examples["drilling_short"]
334
 
335
  def get_example(key, source_lang):
336
- examples = get_examples(source_lang)
337
  return examples[key]
338
 
 
339
  custom_css = """
340
  .gradio-container {
341
  max-width: 1100px !important;
342
  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
343
  }
344
- .main-container {
345
- background: #f6f7f8 !important;
346
- padding: 0 !important;
347
- border-radius: 0 !important;
348
- }
349
  .translate-box {
350
  background: white !important;
351
  border-radius: 5px !important;
352
- padding: 0 !important;
353
  box-shadow: 0 2px 4px rgba(0,0,0,0.08) !important;
354
  margin: 20px 0 !important;
355
  }
@@ -358,13 +189,6 @@ custom_css = """
358
  border-bottom: 1px solid #e8eaed !important;
359
  background: #fafafa !important;
360
  }
361
- .lang-selector {
362
- border: none !important;
363
- background: transparent !important;
364
- font-size: 15px !important;
365
- font-weight: 500 !important;
366
- color: #333 !important;
367
- }
368
  .text-area textarea {
369
  border: none !important;
370
  font-size: 17px !important;
@@ -372,12 +196,6 @@ custom_css = """
372
  padding: 20px !important;
373
  min-height: 200px !important;
374
  }
375
- .swap-container {
376
- display: flex !important;
377
- align-items: center !important;
378
- justify-content: center !important;
379
- padding: 20px 0 !important;
380
- }
381
  .swap-btn {
382
  width: 44px !important;
383
  height: 44px !important;
@@ -388,11 +206,6 @@ custom_css = """
388
  box-shadow: 0 1px 3px rgba(0,0,0,0.1) !important;
389
  font-size: 18px !important;
390
  color: #0f6fff !important;
391
- cursor: pointer !important;
392
- }
393
- .swap-btn:hover {
394
- background: #f8f9fa !important;
395
- border-color: #0f6fff !important;
396
  }
397
  .translate-btn {
398
  background: #ff8c00 !important;
@@ -402,10 +215,6 @@ custom_css = """
402
  font-size: 15px !important;
403
  font-weight: 500 !important;
404
  border-radius: 4px !important;
405
- cursor: pointer !important;
406
- }
407
- .translate-btn:hover {
408
- background: #e67e00 !important;
409
  }
410
  .time-info {
411
  text-align: center !important;
@@ -414,34 +223,12 @@ custom_css = """
414
  padding: 10px !important;
415
  font-style: italic !important;
416
  }
417
- .footer-info {
418
- text-align: center !important;
419
- color: #999 !important;
420
- font-size: 13px !important;
421
- padding: 20px !important;
422
- }
423
- .disclaimer {
424
- background: #fff9e6 !important;
425
- border-left: 4px solid #ff8c00 !important;
426
- padding: 15px 20px !important;
427
- margin: 20px 0 !important;
428
- border-radius: 4px !important;
429
- font-size: 13px !important;
430
- color: #666 !important;
431
- }
432
  """
433
 
434
  with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
435
-
436
  gr.HTML("<div style='height: 20px'></div>")
437
-
438
- with gr.Row():
439
- use_terminology = gr.Checkbox(
440
- label="Enable Terminology Enforcement",
441
- value=True,
442
- info=f"Post-processing with {len(TERMINOLOGY_EN_TO_NO)} oil & gas terms"
443
- )
444
-
445
  with gr.Row():
446
  with gr.Column(scale=1):
447
  with gr.Group(elem_classes="translate-box"):
@@ -451,8 +238,7 @@ with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
451
  value="English",
452
  show_label=False,
453
  container=False,
454
- elem_classes="lang-selector",
455
- scale=1
456
  )
457
  input_text = gr.Textbox(
458
  placeholder="Type to translate",
@@ -460,13 +246,12 @@ with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
460
  lines=8,
461
  max_lines=20,
462
  container=False,
463
- elem_classes="text-area"
464
  )
465
-
466
  with gr.Column(scale=0, min_width=100):
467
- with gr.Row(elem_classes="swap-container"):
468
- swap_btn = gr.Button("⇄", elem_classes="swap-btn")
469
-
470
  with gr.Column(scale=1):
471
  with gr.Group(elem_classes="translate-box"):
472
  with gr.Row(elem_classes="lang-header"):
@@ -475,10 +260,9 @@ with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
475
  value="Norwegian",
476
  show_label=False,
477
  container=False,
478
- elem_classes="lang-selector",
479
- scale=1
480
  )
481
- output_text_plain = gr.Textbox(
482
  placeholder="Translation",
483
  show_label=False,
484
  lines=8,
@@ -486,94 +270,64 @@ with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
486
  container=False,
487
  elem_classes="text-area",
488
  interactive=False,
489
- visible=False
490
- )
491
- output_text_html = gr.HTML(
492
- value="<div style='padding: 20px; min-height: 200px; font-size: 17px; line-height: 1.7;'>Translation</div>"
493
  )
494
-
495
  with gr.Row():
496
  translate_btn = gr.Button("Translate", variant="primary", elem_classes="translate-btn", size="lg")
497
-
498
  with gr.Row():
499
- time_display = gr.Textbox(
500
- show_label=False,
501
- container=False,
502
- interactive=False,
503
- elem_classes="time-info"
504
- )
505
-
506
- gr.HTML("<div class='footer-info'>Oil & Gas Translation • English ↔ Norwegian • Terminology Highlighting</div>")
507
-
508
  with gr.Accordion("Example Sentences", open=True):
509
  with gr.Row():
510
- example_text = gr.Textbox(
511
- value=EXAMPLES_EN["drilling_short"],
512
- label="",
513
- lines=3,
514
- max_lines=5,
515
- show_copy_button=True
516
- )
517
  use_example_btn = gr.Button("Use This Example", variant="primary", size="sm")
518
-
519
  with gr.Row():
520
- btn1 = gr.Button("Drilling Short", size="sm")
521
- btn2 = gr.Button("Drilling Long", size="sm")
522
- btn3 = gr.Button("Reservoir Short", size="sm")
523
- btn4 = gr.Button("Reservoir Long", size="sm")
524
- btn5 = gr.Button("Subsea Short", size="sm")
525
-
 
 
 
 
 
526
  with gr.Row():
527
- btn6 = gr.Button("Subsea Long", size="sm")
528
- btn7 = gr.Button("Seismic Short", size="sm")
529
- btn8 = gr.Button("Seismic Long", size="sm")
530
- btn9 = gr.Button("Safety Short", size="sm")
531
- btn10 = gr.Button("Safety Long", size="sm")
532
-
533
- btn1.click(lambda sl: get_example("drilling_short", sl), inputs=[source_lang], outputs=example_text)
534
- btn2.click(lambda sl: get_example("drilling_long", sl), inputs=[source_lang], outputs=example_text)
535
- btn3.click(lambda sl: get_example("reservoir_short", sl), inputs=[source_lang], outputs=example_text)
536
- btn4.click(lambda sl: get_example("reservoir_long", sl), inputs=[source_lang], outputs=example_text)
537
- btn5.click(lambda sl: get_example("subsea_short", sl), inputs=[source_lang], outputs=example_text)
538
- btn6.click(lambda sl: get_example("subsea_long", sl), inputs=[source_lang], outputs=example_text)
539
- btn7.click(lambda sl: get_example("seismic_short", sl), inputs=[source_lang], outputs=example_text)
540
- btn8.click(lambda sl: get_example("seismic_long", sl), inputs=[source_lang], outputs=example_text)
541
- btn9.click(lambda sl: get_example("safety_short", sl), inputs=[source_lang], outputs=example_text)
542
- btn10.click(lambda sl: get_example("safety_long", sl), inputs=[source_lang], outputs=example_text)
543
-
544
  use_example_btn.click(fn=lambda x: x, inputs=example_text, outputs=input_text)
545
-
546
  with gr.Accordion("Upload Text File", open=False):
547
- file_input = gr.File(
548
- label="Upload a .txt file to translate (max 1MB)",
549
- file_types=[".txt"],
550
- type="filepath"
551
- )
552
-
553
- gr.HTML(f"""
554
- <div class='disclaimer'>
555
- <strong>Terminology Enforcement:</strong> {len(TERMINOLOGY_EN_TO_NO)} oil & gas terms with automatic highlighting
556
- <br>
557
- <strong>Privacy & Compliance:</strong> Local inference ensures GDPR compliance
558
- <br>
559
- <strong>Technical Features:</strong> Optimized batch processing with pre-compiled regex patterns
560
- </div>
561
- """)
562
-
563
- source_lang.change(fn=update_example_buttons, inputs=[source_lang], outputs=[example_text])
564
-
565
  translate_btn.click(
566
  fn=translate,
567
- inputs=[input_text, source_lang, target_lang, use_terminology],
568
- outputs=[output_text_plain, output_text_html, time_display]
569
  )
570
-
571
  swap_btn.click(
572
  fn=swap_languages,
573
- inputs=[source_lang, target_lang, input_text, output_text_plain],
574
- outputs=[source_lang, target_lang, input_text, output_text_plain]
575
  )
576
-
577
- file_input.change(fn=load_file, inputs=file_input, outputs=[input_text, output_text_html, time_display])
 
 
578
 
579
  demo.queue().launch()
 
5
  from functools import lru_cache
6
  import os
7
  import time
8
+ import threading
9
  import nltk
 
 
10
 
11
  try:
12
+ nltk.data.find("tokenizers/punkt")
13
  except LookupError:
14
+ nltk.download("punkt")
15
  try:
16
+ nltk.download("punkt_tab")
17
  except:
18
  pass
19
 
 
23
  ADAPTER_EN_TO_NO = os.getenv("ADAPTER_EN_TO_NO", "entropy25/mt_en_no_oil")
24
  ADAPTER_NO_TO_EN = os.getenv("ADAPTER_NO_TO_EN", "entropy25/mt_no_en_oil")
25
 
26
+ MAX_FILE_SIZE = 1024 * 1024
27
+ MAX_TEXT_LENGTH = 10000
28
+ BATCH_SIZE = 10
29
+ NUM_BEAMS = 3
30
+ MAX_LENGTH = 256
31
 
32
+ tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 
33
 
34
  base_model = AutoModelForSeq2SeqLM.from_pretrained(
35
  BASE_MODEL,
36
+ quantization_config=BitsAndBytesConfig(load_in_8bit=True),
37
+ low_cpu_mem_usage=True,
38
  )
39
 
 
40
  model = PeftModel.from_pretrained(base_model, ADAPTER_EN_TO_NO, adapter_name="en_to_no")
41
  model.load_adapter(ADAPTER_NO_TO_EN, adapter_name="no_to_en")
42
  model.eval()
43
 
44
+ adapter_lock = threading.Lock()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
 
46
+ LANG_CONFIG = {
47
+ ("English", "Norwegian"): ("en_to_no", "nob_Latn"),
48
+ ("Norwegian", "English"): ("no_to_en", "eng_Latn"),
49
  }
50
 
51
+ EXAMPLES_EN = {
52
+ "drilling_short": "Mud weight adjusted to 1.82 specific gravity at 3,247 meters depth.",
53
+ "drilling_long": "The drilling operation at well site A-15 encountered unexpected high-pressure zones at 3,247 meters depth, requiring immediate adjustment of mud weight from 1.65 to 1.82 specific gravity to maintain wellbore stability and prevent potential kicks.",
54
+ "reservoir_short": "Permeability is 250 millidarcy with 22 percent porosity.",
55
+ "reservoir_long": "The reservoir shows excellent permeability of 250 millidarcy and porosity of 22 percent based on core analysis, indicating significant hydrocarbon potential with estimated oil saturation of 65 percent.",
56
+ "subsea_short": "Christmas tree rated for 10,000 psi working pressure.",
57
+ "subsea_long": "The subsea production system consists of a vertical Christmas tree rated for 10,000 psi working pressure and 150 degrees Celsius temperature, equipped with redundant safety features including automatic shutdown valves and real-time pressure monitoring systems.",
58
+ "seismic_short": "Structural trap area estimated at 12 square kilometers.",
59
+ "seismic_long": "Seismic data confirms the presence of a structural trap with an estimated area of 12 square kilometers, and productivity tests show stabilized oil production of 3,400 barrels per day at optimization pressure of 2,100 psi.",
60
+ "safety_short": "H2S training required before site access.",
61
+ "safety_long": "Emergency response procedures require all personnel to complete H2S safety training before site access, with breathing apparatus and wind indicators positioned at designated muster points, and immediate evacuation protocols activated when gas detection exceeds 10 ppm concentration levels.",
 
 
 
 
62
  }
63
 
64
+ EXAMPLES_NO = {
65
+ "drilling_short": "Slamvekt justert til 1,82 spesifikk tyngde ved 3 247 meters dybde.",
66
+ "drilling_long": "Boreoperasjonen ved brønnsted A-15 støtte på uventede høytrykksoner ved 3 247 meters dybde, noe som krevde umiddelbar justering av slamvekt fra 1,65 til 1,82 spesifikk tyngde for å opprettholde brønnborestabilitet og forhindre potensielle kicks.",
67
+ "reservoir_short": "Permeabilitet er 250 millidarcy med 22 prosent porøsitet.",
68
+ "reservoir_long": "Reservoaret viser utmerket permeabilitet på 250 millidarcy og porøsitet på 22 prosent basert på kjerneanalyse, noe som indikerer betydelig hydrokarbonpotensial med estimert oljemetning på 65 prosent.",
69
+ "subsea_short": "Juletre dimensjonert for 10 000 psi arbeidstrykk.",
70
+ "subsea_long": "Subsea produksjonssystemet består av et vertikalt juletre dimensjonert for 10 000 psi arbeidstrykk og 150 grader Celsius temperatur, utstyrt med redundante sikkerhetsfunksjoner inkludert automatiske nedstengningsventiler og sanntids trykkmonitorering.",
71
+ "seismic_short": "Strukturell felle estimert til 12 kvadratkilometer.",
72
+ "seismic_long": "Seismiske data bekrefter tilstedeværelsen av en strukturell felle med estimert areal på 12 kvadratkilometer, og produktivitetstester viser stabilisert oljeproduksjon på 3 400 fat per dag ved optimaliseringstrykk på 2 100 psi.",
73
+ "safety_short": "H2S-opplæring påkrevd før tilgang til området.",
74
+ "safety_long": "Nødprosedyrer krever at alt personell fullfører H2S-sikkerhetsopplæring før områdetilgang, med åndedrettsutstyr og vindindikatorer plassert ved utpekte samlingspunkter, og umiddelbare evakueringsprotokoller aktiveres når gassdeteksjon overskrider 10 ppm konsentrasjonsnivå.",
75
+ }
 
76
 
 
 
 
 
 
77
 
78
  @lru_cache(maxsize=512)
79
  def cached_sent_tokenize(text):
80
  return tuple(sent_tokenize(text))
81
 
82
+
83
+ @lru_cache(maxsize=512)
84
+ def translate_cached(text, source_lang, target_lang):
85
+ if source_lang == target_lang or not text.strip():
86
+ return text, 0.0
87
+
88
+ config = LANG_CONFIG.get((source_lang, target_lang))
89
+ if not config:
90
+ return "Unsupported language pair", 0.0
91
+
92
+ adapter_name, tgt_code = config
93
+ start = time.time()
94
+ device = next(model.parameters()).device
95
+ translated_paragraphs = []
96
+
97
+ for paragraph in text.split("\n"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  if not paragraph.strip():
99
+ translated_paragraphs.append("")
100
  continue
101
+
102
  sentences = cached_sent_tokenize(paragraph)
103
+ results = []
104
+
105
  for i in range(0, len(sentences), BATCH_SIZE):
106
+ batch = sentences[i : i + BATCH_SIZE]
 
107
  inputs = tokenizer(
108
  batch,
109
  return_tensors="pt",
110
  padding=True,
111
  truncation=True,
112
+ max_length=MAX_LENGTH,
113
  )
114
+ inputs = {k: v.to(device) for k, v in inputs.items()}
115
+
116
+ with adapter_lock:
117
+ model.set_adapter(adapter_name)
118
+ with torch.inference_mode():
119
+ outputs = model.generate(
120
+ **inputs,
121
+ forced_bos_token_id=tokenizer.convert_tokens_to_ids(tgt_code),
122
+ max_length=MAX_LENGTH,
123
+ num_beams=NUM_BEAMS,
124
+ early_stopping=True,
125
+ )
126
+
127
+ results.extend(tokenizer.batch_decode(outputs, skip_special_tokens=True))
128
+
129
+ translated_paragraphs.append(" ".join(results))
130
+
131
+ return "\n".join(translated_paragraphs), time.time() - start
 
 
 
 
 
132
 
 
 
 
 
133
 
134
+ def translate(text, source_lang, target_lang):
135
+ if not text.strip():
136
+ return "", ""
137
+ if len(text) > MAX_TEXT_LENGTH:
138
+ return f"Error: Text too long (max {MAX_TEXT_LENGTH:,} characters)", ""
139
+
140
  try:
141
+ result, elapsed = translate_cached(text, source_lang, target_lang)
142
+ return result, f"Completed in {elapsed:.2f}s"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  except Exception as e:
144
+ return f"Translation error: {str(e)}", ""
145
+
146
 
147
  def swap_languages(src, tgt, input_txt, output_txt):
148
  return tgt, src, output_txt, input_txt
149
 
150
+
151
  def load_file(file):
152
  if file is None:
153
+ return "", ""
 
154
  try:
155
+ size = os.path.getsize(file.name)
156
+ if size > MAX_FILE_SIZE:
157
+ return "Error: File too large (max 1MB)", ""
158
+ for encoding in ("utf-8", "latin-1"):
159
+ try:
160
+ with open(file.name, "r", encoding=encoding) as f:
161
+ content = f.read()
 
 
 
 
 
162
  if len(content) > MAX_TEXT_LENGTH:
163
+ return f"Error: File content too long (max {MAX_TEXT_LENGTH:,} characters)", ""
164
+ return content, ""
165
+ except UnicodeDecodeError:
166
+ continue
167
+ except (IOError, OSError) as e:
168
+ return f"Error reading file: {str(e)}", ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
 
 
 
 
170
 
171
  def get_example(key, source_lang):
172
+ examples = EXAMPLES_EN if source_lang == "English" else EXAMPLES_NO
173
  return examples[key]
174
 
175
+
176
  custom_css = """
177
  .gradio-container {
178
  max-width: 1100px !important;
179
  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
180
  }
 
 
 
 
 
181
  .translate-box {
182
  background: white !important;
183
  border-radius: 5px !important;
 
184
  box-shadow: 0 2px 4px rgba(0,0,0,0.08) !important;
185
  margin: 20px 0 !important;
186
  }
 
189
  border-bottom: 1px solid #e8eaed !important;
190
  background: #fafafa !important;
191
  }
 
 
 
 
 
 
 
192
  .text-area textarea {
193
  border: none !important;
194
  font-size: 17px !important;
 
196
  padding: 20px !important;
197
  min-height: 200px !important;
198
  }
 
 
 
 
 
 
199
  .swap-btn {
200
  width: 44px !important;
201
  height: 44px !important;
 
206
  box-shadow: 0 1px 3px rgba(0,0,0,0.1) !important;
207
  font-size: 18px !important;
208
  color: #0f6fff !important;
 
 
 
 
 
209
  }
210
  .translate-btn {
211
  background: #ff8c00 !important;
 
215
  font-size: 15px !important;
216
  font-weight: 500 !important;
217
  border-radius: 4px !important;
 
 
 
 
218
  }
219
  .time-info {
220
  text-align: center !important;
 
223
  padding: 10px !important;
224
  font-style: italic !important;
225
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226
  """
227
 
228
  with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
229
+
230
  gr.HTML("<div style='height: 20px'></div>")
231
+
 
 
 
 
 
 
 
232
  with gr.Row():
233
  with gr.Column(scale=1):
234
  with gr.Group(elem_classes="translate-box"):
 
238
  value="English",
239
  show_label=False,
240
  container=False,
241
+ scale=1,
 
242
  )
243
  input_text = gr.Textbox(
244
  placeholder="Type to translate",
 
246
  lines=8,
247
  max_lines=20,
248
  container=False,
249
+ elem_classes="text-area",
250
  )
251
+
252
  with gr.Column(scale=0, min_width=100):
253
+ swap_btn = gr.Button("⇄", elem_classes="swap-btn")
254
+
 
255
  with gr.Column(scale=1):
256
  with gr.Group(elem_classes="translate-box"):
257
  with gr.Row(elem_classes="lang-header"):
 
260
  value="Norwegian",
261
  show_label=False,
262
  container=False,
263
+ scale=1,
 
264
  )
265
+ output_text = gr.Textbox(
266
  placeholder="Translation",
267
  show_label=False,
268
  lines=8,
 
270
  container=False,
271
  elem_classes="text-area",
272
  interactive=False,
 
 
 
 
273
  )
274
+
275
  with gr.Row():
276
  translate_btn = gr.Button("Translate", variant="primary", elem_classes="translate-btn", size="lg")
277
+
278
  with gr.Row():
279
+ time_display = gr.Textbox(show_label=False, container=False, interactive=False, elem_classes="time-info")
280
+
281
+ gr.HTML("<div style='text-align:center;color:#999;font-size:13px;padding:20px'>Oil & Gas Translation — English ↔ Norwegian</div>")
282
+
 
 
 
 
 
283
  with gr.Accordion("Example Sentences", open=True):
284
  with gr.Row():
285
+ example_text = gr.Textbox(value=EXAMPLES_EN["drilling_short"], label="", lines=3, show_copy_button=True)
 
 
 
 
 
 
286
  use_example_btn = gr.Button("Use This Example", variant="primary", size="sm")
287
+
288
  with gr.Row():
289
+ for key, label in [
290
+ ("drilling_short", "Drilling Short"),
291
+ ("drilling_long", "Drilling Long"),
292
+ ("reservoir_short", "Reservoir Short"),
293
+ ("reservoir_long", "Reservoir Long"),
294
+ ("subsea_short", "Subsea Short"),
295
+ ]:
296
+ gr.Button(label, size="sm").click(
297
+ lambda sl, k=key: get_example(k, sl), inputs=[source_lang], outputs=example_text
298
+ )
299
+
300
  with gr.Row():
301
+ for key, label in [
302
+ ("subsea_long", "Subsea Long"),
303
+ ("seismic_short", "Seismic Short"),
304
+ ("seismic_long", "Seismic Long"),
305
+ ("safety_short", "Safety Short"),
306
+ ("safety_long", "Safety Long"),
307
+ ]:
308
+ gr.Button(label, size="sm").click(
309
+ lambda sl, k=key: get_example(k, sl), inputs=[source_lang], outputs=example_text
310
+ )
311
+
 
 
 
 
 
 
312
  use_example_btn.click(fn=lambda x: x, inputs=example_text, outputs=input_text)
313
+
314
  with gr.Accordion("Upload Text File", open=False):
315
+ file_input = gr.File(label="Upload a .txt file (max 1MB)", file_types=[".txt"], type="filepath")
316
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  translate_btn.click(
318
  fn=translate,
319
+ inputs=[input_text, source_lang, target_lang],
320
+ outputs=[output_text, time_display],
321
  )
322
+
323
  swap_btn.click(
324
  fn=swap_languages,
325
+ inputs=[source_lang, target_lang, input_text, output_text],
326
+ outputs=[source_lang, target_lang, input_text, output_text],
327
  )
328
+
329
+ file_input.change(fn=load_file, inputs=file_input, outputs=[input_text, time_display])
330
+
331
+ source_lang.change(fn=lambda sl: get_example("drilling_short", sl), inputs=[source_lang], outputs=[example_text])
332
 
333
  demo.queue().launch()