File size: 1,842 Bytes
eddaea3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | import requests
from bs4 import BeautifulSoup
from fpdf import FPDF
from huggingface_hub import HfApi, upload_file
import os
HF_TOKEN = os.environ.get("HF_TOKEN", "")
DATASET_REPO = "sosa123454321/Notary-PDF-Dataset"
def scrape_article_to_pdf(url, output_name):
print(f"Scraping {url}...")
try:
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
# Extract main text
for script in soup(["script", "style"]):
script.decompose()
text = soup.get_text()
lines = (line.strip() for line in text.splitlines())
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
clean_text = '\n'.join(chunk for chunk in chunks if chunk)
# Create PDF
pdf = FPDF()
pdf.add_page()
pdf.set_font("Arial", size=10)
# Handle non-latin characters by replacing them for the demo
# For full Persian support, a .ttf font must be loaded using pdf.add_font()
pdf.multi_cell(0, 10, txt=clean_text.encode('latin-1', 'replace').decode('latin-1'))
pdf_path = f"{output_name}.pdf"
pdf.output(pdf_path)
print(f"Saved to {pdf_path}")
# Upload to HF
print(f"Uploading to {DATASET_REPO}...")
upload_file(
path_or_fileobj=pdf_path,
path_in_repo=f"documents/{pdf_path}",
repo_id=DATASET_REPO,
repo_type="dataset",
token=HF_TOKEN
)
print("Upload successful!")
return pdf_path
except Exception as e:
print(f"Error: {e}")
return None
if __name__ == "__main__":
# Example usage
test_url = "https://www.notary662th.ir/induction-manual"
scrape_article_to_pdf(test_url, "notary_induction_persian")
|