Instructions to use openpecha/aligner with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use openpecha/aligner with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSeq2SeqLM tokenizer = AutoTokenizer.from_pretrained("openpecha/aligner") model = AutoModelForSeq2SeqLM.from_pretrained("openpecha/aligner", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Update handler.py
Browse files- handler.py +7 -7
handler.py
CHANGED
|
@@ -215,28 +215,28 @@ def TemporaryDirectory():
|
|
| 215 |
|
| 216 |
def download_file(s3_public_url: str, output_fn: str) -> Path:
|
| 217 |
"""Download file from a public S3 bucket URL and ensure UTF-8 encoding."""
|
| 218 |
-
|
|
|
|
| 219 |
|
| 220 |
# Download the file as binary data
|
| 221 |
with requests.get(s3_public_url, stream=True) as r:
|
| 222 |
r.raise_for_status()
|
| 223 |
-
with open(
|
| 224 |
for chunk in r.iter_content(chunk_size=8192):
|
| 225 |
f.write(chunk)
|
| 226 |
|
| 227 |
# Read the downloaded file and ensure UTF-8 encoding
|
| 228 |
-
with open(
|
| 229 |
content = f.read()
|
| 230 |
|
| 231 |
# Write the content back with correct encoding
|
| 232 |
-
with open(
|
| 233 |
f.write(content)
|
| 234 |
|
| 235 |
# Clean up: remove the temporary file
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
return Path(output_fn)
|
| 239 |
|
|
|
|
| 240 |
def _run_align_script(bo_fn, en_fn, output_dir):
|
| 241 |
start = time.time()
|
| 242 |
cmd = [str(ALIGNER_SCRIPT_PATH), str(bo_fn), str(en_fn), str(output_dir)]
|
|
|
|
| 215 |
|
| 216 |
def download_file(s3_public_url: str, output_fn: str) -> Path:
|
| 217 |
"""Download file from a public S3 bucket URL and ensure UTF-8 encoding."""
|
| 218 |
+
output_path = Path(output_fn)
|
| 219 |
+
temp_path = output_path.with_suffix('.temp')
|
| 220 |
|
| 221 |
# Download the file as binary data
|
| 222 |
with requests.get(s3_public_url, stream=True) as r:
|
| 223 |
r.raise_for_status()
|
| 224 |
+
with open(temp_path, "wb") as f:
|
| 225 |
for chunk in r.iter_content(chunk_size=8192):
|
| 226 |
f.write(chunk)
|
| 227 |
|
| 228 |
# Read the downloaded file and ensure UTF-8 encoding
|
| 229 |
+
with open(temp_path, 'r', encoding='utf-8', errors='replace') as f:
|
| 230 |
content = f.read()
|
| 231 |
|
| 232 |
# Write the content back with correct encoding
|
| 233 |
+
with open(output_path, 'w', encoding='utf-8') as f:
|
| 234 |
f.write(content)
|
| 235 |
|
| 236 |
# Clean up: remove the temporary file
|
| 237 |
+
temp_path.unlink()
|
|
|
|
|
|
|
| 238 |
|
| 239 |
+
return output_path
|
| 240 |
def _run_align_script(bo_fn, en_fn, output_dir):
|
| 241 |
start = time.time()
|
| 242 |
cmd = [str(ALIGNER_SCRIPT_PATH), str(bo_fn), str(en_fn), str(output_dir)]
|