| | import os |
| | import streamlit as st |
| | import requests |
| | from streamlit_lottie import st_lottie |
| |
|
| | def main() -> None: |
| |
|
| | |
| |
|
| | def load_lottieurl(lottie_url:str): |
| | r = requests.get(url=lottie_url) |
| | return r.json() if r.status_code == 200 else None |
| |
|
| | def fetch(url): |
| | try: |
| | result = requests.post(url=os.environ.get('scraper-api-endpoint'), json={'url': url}) |
| | return result.json() |
| | except Exception: |
| | return {} |
| | |
| | st.set_page_config(page_title="Article Scraper - Rahul Portfolio Project", page_icon=":spider:", layout="wide") |
| |
|
| | lottie_animation = load_lottieurl(lottie_url="https://assets3.lottiefiles.com/private_files/lf30_UaWyEa.json") |
| |
|
| | |
| | with st.container(): |
| | st.subheader("Article Scraper") |
| | st.title("A Digital News / Article Information Extraction Application") |
| | st.write("A portfolio project developed to showcase my ability in developing Information Extraction Services") |
| | st.write("This service can be utilised in the data collection / curation process of data science workflow") |
| | st.write("[My Website >](https://www.rahulnenavath.co.in/)") |
| | st.subheader(f'About Article Scraper API:') |
| | st.write( |
| | """ |
| | - Article scraper API is deployed on AWS Lambda using AWS ECR container deployment |
| | - CI/CD workflow is implemented via GitHub Actions to push the latest docker build to AWS ECR |
| | - Check out the API codebase on [my GitHub >](https://github.com/RahulNenavath/Article-Scraper) |
| | API Tech Stack: Python, Beautifulsoup, AWS Lambda, AWS ECR, Docker, GitHub Actions (CI/CD) |
| | """ |
| | ) |
| |
|
| | with st.container(): |
| | st.write("---") |
| | left_col, right_col = st.columns(2) |
| |
|
| | with left_col: |
| | st.header("How it works?") |
| | st.write("##") |
| | st.write('**Input**: Article URL') |
| | st.write('**Output**: Extracted Article Information') |
| | st.write( |
| | """ |
| | **Working**: |
| | - Download the HTML content from the given Article URL |
| | - Makes use of BeautifulSoup and extracts content from different HTML tags and ClassNames |
| | - Arrange Information appropriately |
| | - Regex based text cleaning to remove characters like additional spaces, unicodes, tabs, and newline characters |
| | """ |
| | ) |
| | st.warning(f'Note: Web scraping is highly dependent on the Article HTML structure. Hence one might have to further clean the scraped content') |
| | |
| | with right_col: |
| | st_lottie(lottie_animation, height=500) |
| |
|
| | with st.form("my_form"): |
| | article_url = st.text_input("Article URL", value="", key="article_url") |
| |
|
| | submitted = st.form_submit_button("Submit") |
| |
|
| | if submitted: |
| | with st.spinner('Scraping Information ...'): |
| | data = fetch(url=article_url) |
| | |
| | if data: |
| | st.success("Request is Successful") |
| | content = data.get("scraped_content") |
| | st.write("---") |
| | st.subheader(f'Extracted Article Information') |
| | st.write(f"**Article Title:** {content.get('article_title')}") |
| | st.write(f"**Author:** {content.get('author')}") |
| | st.write(f"**Published Date:** {content.get('publish_date')}") |
| | st.write(f"**Description:** {content.get('description')}") |
| | st.write(f"**Content:** {content.get('article_content')}") |
| | st.write(f"**Article URL:** {content.get('article_url')}") |
| | st.write(f"**Canonical URL:** {content.get('canonical_url')}") |
| | st.write(f"**Publisher Name:** {content.get('publisher_name')}") |
| | st.write(f"**Article Image:** {content.get('image')}") |
| | st.write(f"**Article Keywords:** {content.get('keywords')}") |
| | st.write(f"**Video URL:** {content.get('video_url')}") |
| | st.write(f"**Audio URL:** {content.get('audio_url')}") |
| | else: |
| | st.error("Request Failed") |
| |
|
| |
|
| | if __name__ == "__main__": |
| | main() |
