Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| from transformers import ( | |
| AutoModelForSeq2SeqLM, | |
| AutoTokenizer, | |
| pipeline, | |
| ) | |
| import torch | |
| import nltk | |
| from nltk.tokenize import sent_tokenize, word_tokenize | |
| from nltk.corpus import stopwords | |
| from nltk.stem import WordNetLemmatizer | |
| from bs4 import BeautifulSoup | |
| import requests | |
| import PyPDF2 | |
| import pytesseract | |
| from PIL import Image | |
| import re | |
| import time | |
| from youtube_transcript_api import YouTubeTranscriptApi | |
| import spacy | |
| import logging | |
| import numpy as np | |
| from datetime import datetime | |
| import warnings | |
| warnings.filterwarnings("ignore") | |
| # Configure logging | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format='%(asctime)s - %(levelname)s - %(message)s', | |
| handlers=[ | |
| logging.StreamHandler() | |
| ] | |
| ) | |
| # Download NLTK data at startup | |
| nltk.download('punkt', quiet=True) | |
| nltk.download('stopwords', quiet=True) | |
| nltk.download('wordnet', quiet=True) | |
| nltk.download('averaged_perceptron_tagger', quiet=True) | |
| class FlashcardGenerator: | |
| def __init__(self): | |
| """Initialize the FlashcardGenerator with advanced AI models""" | |
| self.device = "cuda" if torch.cuda.is_available() else "cpu" | |
| logging.info(f"Using device: {self.device}") | |
| # Initialize AI models | |
| self.init_models() | |
| # Initialize spaCy | |
| self.nlp = spacy.load("en_core_web_sm") | |
| self.lemmatizer = WordNetLemmatizer() | |
| self.stop_words = set(stopwords.words('english')) | |
| def init_models(self): | |
| """Initialize various AI models for different tasks""" | |
| try: | |
| # T5 model for question generation | |
| self.question_model = AutoModelForSeq2SeqLM.from_pretrained( | |
| "google/flan-t5-large" | |
| ).to(self.device) | |
| self.question_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large") | |
| # BART model for summarization | |
| self.summarizer = pipeline( | |
| 'summarization', | |
| model="facebook/bart-large-cnn", | |
| device=0 if self.device == "cuda" else -1 | |
| ) | |
| # Use same T5 model for answer generation to save memory | |
| self.answer_model = self.question_model | |
| self.answer_tokenizer = self.question_tokenizer | |
| logging.info("Successfully initialized all AI models") | |
| except Exception as e: | |
| logging.error(f"Error initializing models: {str(e)}") | |
| raise | |
| def preprocess_text(self, text): | |
| """Advanced text preprocessing""" | |
| try: | |
| # Basic cleaning | |
| text = re.sub(r'\s+', ' ', text) | |
| text = re.sub(r'[^\w\s.,?!]', '', text) | |
| # SpaCy processing | |
| doc = self.nlp(text) | |
| # Remove named entities for better question generation | |
| text_without_ents = ' '.join([token.text if not token.ent_type_ else '[ENT]' | |
| for token in doc]) | |
| # Lemmatization | |
| words = word_tokenize(text_without_ents) | |
| lemmatized = [self.lemmatizer.lemmatize(word) for word in words] | |
| return ' '.join(lemmatized) | |
| except Exception as e: | |
| logging.error(f"Error in text preprocessing: {str(e)}") | |
| return text | |
| def generate_questions(self, text, num_questions=5): | |
| """Generate high-quality questions using T5-large model""" | |
| try: | |
| doc = self.nlp(text) | |
| important_sentences = [] | |
| # Extract important sentences based on named entities and noun chunks | |
| for sent in doc.sents: | |
| if (len(sent.ents) > 0 or | |
| len(list(sent.noun_chunks)) > 2 or | |
| any(token.pos_ in ['VERB', 'NUM'] for token in sent)): | |
| important_sentences.append(sent.text) | |
| questions = [] | |
| for sent in important_sentences[:num_questions]: | |
| inputs = self.question_tokenizer( | |
| f"Generate a question: {sent}", | |
| return_tensors="pt", | |
| max_length=512, | |
| truncation=True | |
| ).to(self.device) | |
| outputs = self.question_model.generate( | |
| inputs.input_ids, | |
| max_length=64, | |
| num_beams=4, | |
| length_penalty=1.0, | |
| early_stopping=True | |
| ) | |
| question = self.question_tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| questions.append({"question": question, "context": sent}) | |
| return questions | |
| except Exception as e: | |
| logging.error(f"Error in question generation: {str(e)}") | |
| return [] | |
| def generate_answers(self, questions): | |
| """Generate detailed answers using T5-large model""" | |
| try: | |
| qa_pairs = [] | |
| for q in questions: | |
| input_text = f"Provide an answer: Question: {q['question']} Context: {q['context']}" | |
| inputs = self.answer_tokenizer( | |
| input_text, | |
| return_tensors="pt", | |
| max_length=512, | |
| truncation=True | |
| ).to(self.device) | |
| outputs = self.answer_model.generate( | |
| inputs.input_ids, | |
| max_length=128, | |
| num_beams=4, | |
| length_penalty=1.0, | |
| early_stopping=True | |
| ) | |
| answer = self.answer_tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| qa_pairs.append({ | |
| "question": q['question'], | |
| "answer": answer, | |
| "context": q['context'] | |
| }) | |
| return qa_pairs | |
| except Exception as e: | |
| logging.error(f"Error in answer generation: {str(e)}") | |
| return [] | |
| def extract_keywords(self, text): | |
| """Extract important keywords using spaCy""" | |
| doc = self.nlp(text) | |
| keywords = [] | |
| # Extract named entities | |
| entities = [ent.text for ent in doc.ents] | |
| # Extract important noun phrases | |
| noun_phrases = [chunk.text for chunk in doc.noun_chunks] | |
| # Extract important verbs | |
| verbs = [token.lemma_ for token in doc if token.pos_ == 'VERB'] | |
| keywords = list(set(entities + noun_phrases + verbs)) | |
| return keywords[:10] # Return top 10 keywords | |
| def generate_summary(self, text): | |
| """Generate comprehensive summary""" | |
| try: | |
| chunks = [text[i:i+1000] for i in range(0, len(text), 1000)] | |
| summaries = [] | |
| for chunk in chunks: | |
| summary = self.summarizer( | |
| chunk, | |
| max_length=150, | |
| min_length=40, | |
| do_sample=False, | |
| num_beams=4 | |
| )[0]['summary_text'] | |
| summaries.append(summary) | |
| return ' '.join(summaries) | |
| except Exception as e: | |
| logging.error(f"Error in summary generation: {str(e)}") | |
| return "Summary generation failed." | |
| def process_content(self, text, num_cards=5): | |
| """Process content and generate enhanced flashcards""" | |
| try: | |
| # Preprocess text | |
| cleaned_text = self.preprocess_text(text) | |
| # Generate summary | |
| summary = self.generate_summary(cleaned_text) | |
| # Extract keywords | |
| keywords = self.extract_keywords(cleaned_text) | |
| # Generate questions | |
| questions = self.generate_questions(cleaned_text, num_cards) | |
| # Generate answers | |
| qa_pairs = self.generate_answers(questions) | |
| # Format output | |
| output = self.format_output(summary, keywords, qa_pairs) | |
| return output | |
| except Exception as e: | |
| logging.error(f"Error in content processing: {str(e)}") | |
| return f"Error processing content: {str(e)}" | |
| def format_output(self, summary, keywords, qa_pairs): | |
| """Format output in an enhanced markdown structure""" | |
| timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") | |
| output = f"# Generated Flashcards ({timestamp})\n\n" | |
| # Add summary section | |
| output += "## Summary\n" | |
| output += f"{summary}\n\n" | |
| # Add keywords section | |
| output += "## Key Concepts\n" | |
| output += ", ".join(keywords) + "\n\n" | |
| # Add flashcards section | |
| output += "## Flashcards\n\n" | |
| for i, qa in enumerate(qa_pairs, 1): | |
| output += f"### Card {i}\n" | |
| output += f"**Question:** {qa['question']}\n\n" | |
| output += f"**Answer:** {qa['answer']}\n\n" | |
| output += f"*Context:* _{qa['context']}_\n\n" | |
| output += "---\n\n" | |
| return output | |
| def create_interface(): | |
| """Create the Gradio interface""" | |
| generator = FlashcardGenerator() | |
| description = """ | |
| This advanced tool generates high-quality flashcards using state-of-the-art AI models: | |
| - Uses FLAN-T5-Large for question and answer generation | |
| - Uses BART-Large for summarization | |
| - Implements spaCy for natural language processing | |
| Features: | |
| - Intelligent question generation based on content importance | |
| - Detailed, contextual answers | |
| - Content summarization | |
| - Key concept extraction | |
| - Named entity recognition | |
| - Advanced text preprocessing | |
| Input Options: | |
| - Plain text | |
| - Website URL | |
| - PDF documents | |
| - Images with text (OCR) | |
| **Note:** For best results, provide clear and structured content. | |
| """ | |
| def process_input(input_type, text_input, file_input, num_cards): | |
| """Process input and generate flashcards""" | |
| try: | |
| # Extract text based on input type | |
| if input_type == 'Text': | |
| text = text_input | |
| elif input_type == 'PDF': | |
| # Process PDF | |
| if file_input is not None: | |
| reader = PyPDF2.PdfReader(file_input) | |
| text = "" | |
| for page in reader.pages: | |
| text += page.extract_text() + " " | |
| else: | |
| return "No PDF file uploaded." | |
| elif input_type == 'Image': | |
| # Process Image | |
| if file_input is not None: | |
| image = Image.open(file_input) | |
| text = pytesseract.image_to_string(image) | |
| else: | |
| return "No image file uploaded." | |
| elif input_type == 'Website URL': | |
| # Process URL | |
| response = requests.get(text_input) | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| text = ' '.join([p.get_text() for p in soup.find_all('p')]) | |
| else: | |
| return "Invalid input type" | |
| # Generate flashcards | |
| return generator.process_content(text, num_cards) | |
| except Exception as e: | |
| logging.error(f"Error processing input: {str(e)}") | |
| return f"Error processing input: {str(e)}" | |
| # Create interface | |
| iface = gr.Interface( | |
| fn=process_input, | |
| inputs=[ | |
| gr.Radio( | |
| choices=['Text', 'Website URL', 'PDF', 'Image'], | |
| label='Select Input Type' | |
| ), | |
| gr.Textbox( | |
| lines=5, | |
| placeholder='Enter text or URL here...', | |
| label='Text/URL Input' | |
| ), | |
| gr.File( | |
| label='File Upload (for PDF or Image)' | |
| ), | |
| gr.Slider( | |
| minimum=1, | |
| maximum=10, | |
| value=5, | |
| step=1, | |
| label='Number of Flashcards to Generate' | |
| ) | |
| ], | |
| outputs=gr.Markdown(label='Generated Flashcards'), | |
| title="Professional Flashcard Generator", | |
| description=description, | |
| theme="default" | |
| ) | |
| return iface | |
| # Create and launch the interface | |
| iface = create_interface() | |
| iface.launch() |