Spaces:
Sleeping
Sleeping
Update Milestone5API.py
Browse files- Milestone5API.py +38 -0
Milestone5API.py
CHANGED
|
@@ -46,3 +46,41 @@ def download_video_transcript(video_url):
|
|
| 46 |
except Exception as e:
|
| 47 |
print("Error:", e)
|
| 48 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
except Exception as e:
|
| 47 |
print("Error:", e)
|
| 48 |
return None
|
| 49 |
+
from transformers import MarianMTModel, MarianTokenizer
|
| 50 |
+
import os
|
| 51 |
+
|
| 52 |
+
def translate_text_file(input_text_path, output_text_path, source_lang='en', target_lang='fr', batch_size=8):
|
| 53 |
+
model_name = "Helsinki-NLP/opus-mt-en-fr"
|
| 54 |
+
|
| 55 |
+
model = MarianMTModel.from_pretrained(model_name)
|
| 56 |
+
tokenizer = MarianTokenizer.from_pretrained(model_name)
|
| 57 |
+
|
| 58 |
+
def translate_batch(model, tokenizer, sentences):
|
| 59 |
+
sentences = [f"{source_lang}: {sentence}" for sentence in sentences]
|
| 60 |
+
|
| 61 |
+
input_ids = tokenizer(sentences, return_tensors="pt", padding=True, truncation=True)["input_ids"]
|
| 62 |
+
translation_ids = model.generate(input_ids)
|
| 63 |
+
translated_texts = tokenizer.batch_decode(translation_ids, skip_special_tokens=True)
|
| 64 |
+
|
| 65 |
+
return translated_texts
|
| 66 |
+
|
| 67 |
+
def read_text_from_file(file_path):
|
| 68 |
+
with open(file_path, 'r', encoding='utf-8') as file:
|
| 69 |
+
text = file.readlines()
|
| 70 |
+
return text
|
| 71 |
+
|
| 72 |
+
def write_text_to_file(file_path, translated_texts):
|
| 73 |
+
with open(file_path, 'w', encoding='utf-8') as file:
|
| 74 |
+
file.writelines([f"{line}\n" for line in translated_texts])
|
| 75 |
+
|
| 76 |
+
input_lines = read_text_from_file(input_text_path)
|
| 77 |
+
translated_lines = []
|
| 78 |
+
|
| 79 |
+
for i in range(0, len(input_lines), batch_size):
|
| 80 |
+
batch = input_lines[i:i + batch_size]
|
| 81 |
+
translated_batch = translate_batch(model, tokenizer, batch)
|
| 82 |
+
translated_lines.extend(translated_batch)
|
| 83 |
+
|
| 84 |
+
write_text_to_file(output_text_path, translated_lines)
|
| 85 |
+
return translated_lines
|
| 86 |
+
|