Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| import re | |
| from newspaper import Article | |
| import requests | |
| import io | |
| import os | |
| import requests | |
| from bs4 import BeautifulSoup | |
| from transformers import pipeline | |
| # Sumy and NLTK imports | |
| from nltk.tokenize import sent_tokenize | |
| from sumy.parsers.plaintext import PlaintextParser | |
| from sumy.nlp.tokenizers import Tokenizer | |
| from sumy.summarizers.lsa import LsaSummarizer | |
| from sumy.nlp.stemmers import Stemmer | |
| from sumy.utils import get_stop_words | |
| classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli") | |
| # -------- Summary Cleaning and Extraction -------- # | |
| def preprocess_text(text): | |
| if not isinstance(text, str): | |
| return "" | |
| text = re.sub(r'http\S+', ' ', text) | |
| lines = text.splitlines() | |
| kept = [] | |
| for line in lines: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| if re.match(r'By\s+\S+', line): continue | |
| if re.search(r'\bFollow\b', line): continue | |
| if re.search(r'\d+\s+min\s+read', line, flags=re.IGNORECASE): continue | |
| if re.search(r'\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}\b', line): continue | |
| if line.lower().startswith(( | |
| "read more", "continue reading", "more from medium", | |
| "about the author", "related stories", "you might also like" | |
| )): continue | |
| if line.isupper() and len(line.split()) > 3: | |
| continue | |
| kept.append(line) | |
| text = "\n".join(kept) | |
| text = re.sub(r'[^\w\s.,!?;:]', ' ', text) | |
| text = re.sub(r'\s+', ' ', text).strip() | |
| sents = sent_tokenize(text) | |
| return ' '.join(dict.fromkeys([s for s in sents if len(s.split()) > 3])) | |
| def summarize_with_sumy_auto(text, summary_frac=0.2, min_sentences=3, max_sentences=10): | |
| if not isinstance(text, str): | |
| return "" | |
| cleaned = preprocess_text(text) | |
| orig = sent_tokenize(cleaned) | |
| total = len(orig) | |
| if total <= min_sentences: | |
| return ' '.join(orig) | |
| n = max(min_sentences, min(max_sentences, int(total * summary_frac))) | |
| parser = PlaintextParser.from_string(cleaned, Tokenizer("english")) | |
| stemmer = Stemmer("english") | |
| summarizer = LsaSummarizer(stemmer) | |
| summarizer.stop_words = get_stop_words("english") | |
| sents = summarizer(parser.document, n) | |
| return ' '.join(str(s) for s in sents) | |
| # -------- Utility Functions -------- # | |
| def check_url_status(url: str, timeout: int = 5) -> str: | |
| try: | |
| resp = requests.head(url, allow_redirects=True, timeout=timeout) | |
| if resp.status_code == 405: | |
| resp = requests.get(url, allow_redirects=True, timeout=timeout) | |
| return 'Workable' if resp.status_code == 200 else f'Not Workable ({resp.status_code})' | |
| except requests.RequestException: | |
| return 'Not Workable' | |
| def detect_keywords_and_score(content, url): | |
| keywords = [] | |
| score = 0 | |
| imarticus_found = False | |
| pga_link_found = False | |
| pga_link = "https://imarticus.org/postgraduate-program-in-data-science-analytics/" | |
| if content and re.search(r'imarticus', content, re.IGNORECASE): | |
| keywords.append('Imarticus') | |
| imarticus_found = True | |
| if pga_link in content or pga_link in url: | |
| pga_link_found = True | |
| if content and re.search(r'post graduate', content, re.IGNORECASE): | |
| keywords.append('post graduate') | |
| if imarticus_found: | |
| score = 5 if pga_link_found else 3 | |
| return keywords, score | |
| else: | |
| return [], 0 | |
| def detect_code_snippet(content): | |
| if not content: | |
| return False | |
| code_markers = [ | |
| r'```', r'<code>', r'</code>', r'\n ', r'\t', | |
| r'def ', r'class ', r'\{', r'\}', r';', r'\(', r'\)', r'import ', r'from ', r'print\(' | |
| ] | |
| for marker in code_markers: | |
| if re.search(marker, content): | |
| return True | |
| return False | |
| # ------ Originality Check -----------# | |
| def extract_blog_text(url): | |
| headers = {'User-Agent': 'Mozilla/5.0'} | |
| response = requests.get(url, headers=headers) | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| paragraphs = soup.find_all('p') | |
| return ' '.join([p.get_text() for p in paragraphs]) | |
| def get_ai_generated_score(url, classifier=classifier): | |
| text = extract_blog_text(url) | |
| #classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli") | |
| labels = ["Human-written", "AI-generated"] | |
| result = classifier(text, candidate_labels=labels) | |
| scores = dict(zip(result['labels'], result['scores'])) | |
| return scores.get("AI-generated", 0.0) | |
| # -------- Main Summary Extraction -------- # | |
| def extract_summary(file): | |
| df = pd.read_excel(file) | |
| total_blogs = len(df) | |
| imarticus_count = 0 | |
| code_snippet_count = 0 | |
| filtered_rows = [] | |
| full_analysis = [] | |
| for _, row in df.iterrows(): | |
| url = row.get("Blog Link(Medium link)") or row.get("URL") or row.get("url") | |
| if pd.isna(url): | |
| continue | |
| status = check_url_status(url) | |
| name = row.get("Participant") or row.get("Name") | |
| if not name: | |
| continue | |
| centre = row.get("Centre") or row.get("Center") | |
| if not centre: | |
| continue | |
| #originality = get_ai_generated_score(url) | |
| try: | |
| article = Article(url) | |
| article.download() | |
| article.parse() | |
| title = article.title | |
| content = article.text | |
| if len(content.strip()) == 0: | |
| continue | |
| summary = summarize_with_sumy_auto(content) | |
| keywords, score = detect_keywords_and_score(content, url) | |
| code_snippet = detect_code_snippet(content) | |
| if score > 0: | |
| imarticus_count += 1 | |
| if code_snippet: | |
| code_snippet_count += 1 | |
| filtered_rows.append({ | |
| "Participant": name, | |
| "Centre": centre, | |
| "URL": url, | |
| "Status": status, | |
| "Title": title, | |
| "Content": content, | |
| "Summary": summary, | |
| "Identified_Keywords": ', '.join(keywords) if keywords else "None", | |
| "Code_Snippet": code_snippet, | |
| "Score": score | |
| # "Originality(AI-Score)": originality | |
| }) | |
| full_analysis.append({ | |
| "Participant": name, | |
| "Centre": centre, | |
| "URL": url, | |
| "Title": title, | |
| "Identified_Keywords": ', '.join(keywords) if keywords else "None", | |
| "Code_Snippet": code_snippet, | |
| "Score": score, | |
| "Summary": summary, | |
| "Status": status | |
| # "Originality(AI-Score)": originality | |
| }) | |
| except Exception as e: | |
| print(f"Error processing {url}: {e}") | |
| continue | |
| filtered_df = pd.DataFrame(filtered_rows) | |
| full_df = pd.DataFrame(full_analysis) | |
| return ( | |
| str(total_blogs), | |
| str(code_snippet_count), | |
| str(imarticus_count), | |
| filtered_df, | |
| full_df | |
| ) | |
| def filter_analysis(full_df, status_filter, score_filter): | |
| df = full_df.copy() | |
| if status_filter != "All": | |
| df = df[df["Status"].str.contains(status_filter)] | |
| if score_filter != "All": | |
| df = df[df["Score"] == int(score_filter)] | |
| df = df[["Title", "Identified_Keywords", "Code_Snippet", "Score", "Summary"]] | |
| return df | |
| def download_file(full_df): | |
| if full_df is None or full_df.empty: | |
| print("No data to download.") | |
| return None | |
| output_dir = "./output" | |
| os.makedirs(output_dir, exist_ok=True) | |
| file_path = os.path.join(output_dir, "Full_Analysis.xlsx") | |
| try: | |
| full_df.to_excel(file_path, index=False) | |
| except Exception as e: | |
| print(f"Error saving file: {e}") | |
| return None | |
| return file_path | |
| def trigger_download(full_df): | |
| path = download_file(full_df) | |
| return path, gr.update(visible=True) if path else gr.update(visible=False) | |
| # -------- Gradio UI -------- # | |
| with gr.Blocks(css=""" | |
| .sidebar { background-color: #00664d; color: white; padding: 20px; height: 100%; border-radius: 10px; } | |
| .sidebar label, .sidebar h2, .sidebar h3, .sidebar span, .sidebar p { color: black !important; } | |
| .main-content { padding: 20px; background-color: #ffffff; border-radius: 10px; } | |
| h1, h3 { color: #00664d; } | |
| @media (min-width: 1024px) { | |
| .gr-block.gr-box { max-width: 1000px; margin: auto; } | |
| } | |
| """) as demo: | |
| with gr.Row(): | |
| with gr.Column(scale=1, elem_classes="sidebar"): | |
| gr.Markdown("## 📅 Upload & Filter", elem_id="sidebar-title") | |
| file_input = gr.File(label="Upload Excel File (.xlsx)", file_types=[".xlsx"]) | |
| analyze_btn = gr.Button("Run Summary") | |
| gr.Markdown("## 🔎 Filter") | |
| status_filter = gr.Dropdown(["All", "Workable", "Not Workable"], label="Status", value="All") | |
| score_filter = gr.Dropdown(["All", "0", "3", "5"], label="Score", value="All") | |
| download_btn = gr.Button("Download Full Analysis") | |
| download_file_output = gr.File(label="") | |
| with gr.Column(scale=3, elem_classes="main-content"): | |
| gr.Markdown("<h1>📊 Blog Evaluator </h1>") | |
| gr.Markdown("<h3>Analyze blog URLs for educational content, keywords, and coding examples</h3>") | |
| with gr.Row(): | |
| total_blogs = gr.Textbox(label="Total Blogs", interactive=False) | |
| code_snippets = gr.Textbox(label="Blogs with Code Snippets", interactive=False) | |
| imarticus_hits = gr.Textbox(label="Blogs with 'Imarticus' Mentions", interactive=False) | |
| gr.Markdown("### 📋 Filtered Results Table") | |
| full_table = gr.Dataframe( | |
| headers=["Participant", "Centre","URL","Status","Title","Content","Summary","Identified_Keywords", "Code_Snippet", "Score"], | |
| interactive=False, | |
| datatype=["str", "str", "str", "str", "str","str","str","str","bool","number"], | |
| row_count=10, | |
| col_count=(10, "fixed") | |
| ) | |
| gr.Markdown("### 📋 Full Analyzed Blog Data Table") | |
| filtered_table = gr.Dataframe(headers=["URL", "Status", "Title", "Content", "Summary"], interactive=False) | |
| state_full_df = gr.State() | |
| def analyze(file): | |
| total, codes, imarts, filtered_df, full_df = extract_summary(file) | |
| return total, codes, imarts, filtered_df, full_df.values.tolist(), full_df | |
| def apply_filters(full_df, status, score): | |
| df = filter_analysis(full_df, status, score) | |
| return df.values.tolist() | |
| analyze_btn.click( | |
| fn=analyze, | |
| inputs=file_input, | |
| outputs=[total_blogs, code_snippets, imarticus_hits, filtered_table, full_table, state_full_df] | |
| ) | |
| status_filter.change( | |
| fn=apply_filters, | |
| inputs=[state_full_df, status_filter, score_filter], | |
| outputs=full_table | |
| ) | |
| score_filter.change( | |
| fn=apply_filters, | |
| inputs=[state_full_df, status_filter, score_filter], | |
| outputs=full_table | |
| ) | |
| download_btn.click( | |
| fn=download_file, | |
| inputs=state_full_df, | |
| outputs=download_file_output | |
| ) | |
| demo.launch(share=True) | |