import gradio as gr from transformers import ( AutoModelForSeq2SeqLM, AutoTokenizer, pipeline, ) import torch import nltk from nltk.tokenize import sent_tokenize, word_tokenize from nltk.corpus import stopwords from nltk.stem import WordNetLemmatizer from bs4 import BeautifulSoup import requests import PyPDF2 import pytesseract from PIL import Image import re import time from youtube_transcript_api import YouTubeTranscriptApi import spacy import logging import numpy as np from datetime import datetime import warnings warnings.filterwarnings("ignore") # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler() ] ) # Download NLTK data at startup nltk.download('punkt', quiet=True) nltk.download('stopwords', quiet=True) nltk.download('wordnet', quiet=True) nltk.download('averaged_perceptron_tagger', quiet=True) class FlashcardGenerator: def __init__(self): """Initialize the FlashcardGenerator with advanced AI models""" self.device = "cuda" if torch.cuda.is_available() else "cpu" logging.info(f"Using device: {self.device}") # Initialize AI models self.init_models() # Initialize spaCy self.nlp = spacy.load("en_core_web_sm") self.lemmatizer = WordNetLemmatizer() self.stop_words = set(stopwords.words('english')) def init_models(self): """Initialize various AI models for different tasks""" try: # T5 model for question generation self.question_model = AutoModelForSeq2SeqLM.from_pretrained( "google/flan-t5-large" ).to(self.device) self.question_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large") # BART model for summarization self.summarizer = pipeline( 'summarization', model="facebook/bart-large-cnn", device=0 if self.device == "cuda" else -1 ) # Use same T5 model for answer generation to save memory self.answer_model = self.question_model self.answer_tokenizer = self.question_tokenizer logging.info("Successfully initialized all AI models") except Exception as e: logging.error(f"Error initializing models: {str(e)}") raise def preprocess_text(self, text): """Advanced text preprocessing""" try: # Basic cleaning text = re.sub(r'\s+', ' ', text) text = re.sub(r'[^\w\s.,?!]', '', text) # SpaCy processing doc = self.nlp(text) # Remove named entities for better question generation text_without_ents = ' '.join([token.text if not token.ent_type_ else '[ENT]' for token in doc]) # Lemmatization words = word_tokenize(text_without_ents) lemmatized = [self.lemmatizer.lemmatize(word) for word in words] return ' '.join(lemmatized) except Exception as e: logging.error(f"Error in text preprocessing: {str(e)}") return text def generate_questions(self, text, num_questions=5): """Generate high-quality questions using T5-large model""" try: doc = self.nlp(text) important_sentences = [] # Extract important sentences based on named entities and noun chunks for sent in doc.sents: if (len(sent.ents) > 0 or len(list(sent.noun_chunks)) > 2 or any(token.pos_ in ['VERB', 'NUM'] for token in sent)): important_sentences.append(sent.text) questions = [] for sent in important_sentences[:num_questions]: inputs = self.question_tokenizer( f"Generate a question: {sent}", return_tensors="pt", max_length=512, truncation=True ).to(self.device) outputs = self.question_model.generate( inputs.input_ids, max_length=64, num_beams=4, length_penalty=1.0, early_stopping=True ) question = self.question_tokenizer.decode(outputs[0], skip_special_tokens=True) questions.append({"question": question, "context": sent}) return questions except Exception as e: logging.error(f"Error in question generation: {str(e)}") return [] def generate_answers(self, questions): """Generate detailed answers using T5-large model""" try: qa_pairs = [] for q in questions: input_text = f"Provide an answer: Question: {q['question']} Context: {q['context']}" inputs = self.answer_tokenizer( input_text, return_tensors="pt", max_length=512, truncation=True ).to(self.device) outputs = self.answer_model.generate( inputs.input_ids, max_length=128, num_beams=4, length_penalty=1.0, early_stopping=True ) answer = self.answer_tokenizer.decode(outputs[0], skip_special_tokens=True) qa_pairs.append({ "question": q['question'], "answer": answer, "context": q['context'] }) return qa_pairs except Exception as e: logging.error(f"Error in answer generation: {str(e)}") return [] def extract_keywords(self, text): """Extract important keywords using spaCy""" doc = self.nlp(text) keywords = [] # Extract named entities entities = [ent.text for ent in doc.ents] # Extract important noun phrases noun_phrases = [chunk.text for chunk in doc.noun_chunks] # Extract important verbs verbs = [token.lemma_ for token in doc if token.pos_ == 'VERB'] keywords = list(set(entities + noun_phrases + verbs)) return keywords[:10] # Return top 10 keywords def generate_summary(self, text): """Generate comprehensive summary""" try: chunks = [text[i:i+1000] for i in range(0, len(text), 1000)] summaries = [] for chunk in chunks: summary = self.summarizer( chunk, max_length=150, min_length=40, do_sample=False, num_beams=4 )[0]['summary_text'] summaries.append(summary) return ' '.join(summaries) except Exception as e: logging.error(f"Error in summary generation: {str(e)}") return "Summary generation failed." def process_content(self, text, num_cards=5): """Process content and generate enhanced flashcards""" try: # Preprocess text cleaned_text = self.preprocess_text(text) # Generate summary summary = self.generate_summary(cleaned_text) # Extract keywords keywords = self.extract_keywords(cleaned_text) # Generate questions questions = self.generate_questions(cleaned_text, num_cards) # Generate answers qa_pairs = self.generate_answers(questions) # Format output output = self.format_output(summary, keywords, qa_pairs) return output except Exception as e: logging.error(f"Error in content processing: {str(e)}") return f"Error processing content: {str(e)}" def format_output(self, summary, keywords, qa_pairs): """Format output in an enhanced markdown structure""" timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") output = f"# Generated Flashcards ({timestamp})\n\n" # Add summary section output += "## Summary\n" output += f"{summary}\n\n" # Add keywords section output += "## Key Concepts\n" output += ", ".join(keywords) + "\n\n" # Add flashcards section output += "## Flashcards\n\n" for i, qa in enumerate(qa_pairs, 1): output += f"### Card {i}\n" output += f"**Question:** {qa['question']}\n\n" output += f"**Answer:** {qa['answer']}\n\n" output += f"*Context:* _{qa['context']}_\n\n" output += "---\n\n" return output def create_interface(): """Create the Gradio interface""" generator = FlashcardGenerator() description = """ This advanced tool generates high-quality flashcards using state-of-the-art AI models: - Uses FLAN-T5-Large for question and answer generation - Uses BART-Large for summarization - Implements spaCy for natural language processing Features: - Intelligent question generation based on content importance - Detailed, contextual answers - Content summarization - Key concept extraction - Named entity recognition - Advanced text preprocessing Input Options: - Plain text - Website URL - PDF documents - Images with text (OCR) **Note:** For best results, provide clear and structured content. """ def process_input(input_type, text_input, file_input, num_cards): """Process input and generate flashcards""" try: # Extract text based on input type if input_type == 'Text': text = text_input elif input_type == 'PDF': # Process PDF if file_input is not None: reader = PyPDF2.PdfReader(file_input) text = "" for page in reader.pages: text += page.extract_text() + " " else: return "No PDF file uploaded." elif input_type == 'Image': # Process Image if file_input is not None: image = Image.open(file_input) text = pytesseract.image_to_string(image) else: return "No image file uploaded." elif input_type == 'Website URL': # Process URL response = requests.get(text_input) soup = BeautifulSoup(response.text, 'html.parser') text = ' '.join([p.get_text() for p in soup.find_all('p')]) else: return "Invalid input type" # Generate flashcards return generator.process_content(text, num_cards) except Exception as e: logging.error(f"Error processing input: {str(e)}") return f"Error processing input: {str(e)}" # Create interface iface = gr.Interface( fn=process_input, inputs=[ gr.Radio( choices=['Text', 'Website URL', 'PDF', 'Image'], label='Select Input Type' ), gr.Textbox( lines=5, placeholder='Enter text or URL here...', label='Text/URL Input' ), gr.File( label='File Upload (for PDF or Image)' ), gr.Slider( minimum=1, maximum=10, value=5, step=1, label='Number of Flashcards to Generate' ) ], outputs=gr.Markdown(label='Generated Flashcards'), title="Professional Flashcard Generator", description=description, theme="default" ) return iface # Create and launch the interface iface = create_interface() iface.launch()