from crewai_tools import ScrapeWebsiteTool
import os

# List of pages you want to scrape
pages = [
    "https://www.sunmarke.com/",
]

all_text = []

for url in pages:
    tool = ScrapeWebsiteTool(website_url=url)
    content = tool.run()
    all_text.append(content)

# Remove duplicates
all_text = list(set(all_text))

# Combine into one string
text = "\n\n".join(all_text)

# Save to file
output_folder = "website"
os.makedirs(output_folder, exist_ok=True)
file_path = os.path.join(output_folder, "output.txt")

with open(file_path, "w", encoding="utf-8") as f:
    f.write("=== Output Start ===\n\n")
    f.write(text.strip())
    f.write("\n\n=== Output End ===\n")

print(f"Text saved to {file_path}")