JaMugen commited on
Commit
28ea672
·
1 Parent(s): 5531cc1

Update Milestone5API.py

Browse files
Files changed (1) hide show
  1. Milestone5API.py +38 -0
Milestone5API.py CHANGED
@@ -46,3 +46,41 @@ def download_video_transcript(video_url):
46
  except Exception as e:
47
  print("Error:", e)
48
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  except Exception as e:
47
  print("Error:", e)
48
  return None
49
+ from transformers import MarianMTModel, MarianTokenizer
50
+ import os
51
+
52
+ def translate_text_file(input_text_path, output_text_path, source_lang='en', target_lang='fr', batch_size=8):
53
+ model_name = "Helsinki-NLP/opus-mt-en-fr"
54
+
55
+ model = MarianMTModel.from_pretrained(model_name)
56
+ tokenizer = MarianTokenizer.from_pretrained(model_name)
57
+
58
+ def translate_batch(model, tokenizer, sentences):
59
+ sentences = [f"{source_lang}: {sentence}" for sentence in sentences]
60
+
61
+ input_ids = tokenizer(sentences, return_tensors="pt", padding=True, truncation=True)["input_ids"]
62
+ translation_ids = model.generate(input_ids)
63
+ translated_texts = tokenizer.batch_decode(translation_ids, skip_special_tokens=True)
64
+
65
+ return translated_texts
66
+
67
+ def read_text_from_file(file_path):
68
+ with open(file_path, 'r', encoding='utf-8') as file:
69
+ text = file.readlines()
70
+ return text
71
+
72
+ def write_text_to_file(file_path, translated_texts):
73
+ with open(file_path, 'w', encoding='utf-8') as file:
74
+ file.writelines([f"{line}\n" for line in translated_texts])
75
+
76
+ input_lines = read_text_from_file(input_text_path)
77
+ translated_lines = []
78
+
79
+ for i in range(0, len(input_lines), batch_size):
80
+ batch = input_lines[i:i + batch_size]
81
+ translated_batch = translate_batch(model, tokenizer, batch)
82
+ translated_lines.extend(translated_batch)
83
+
84
+ write_text_to_file(output_text_path, translated_lines)
85
+ return translated_lines
86
+