deepkansara-123 commited on
Commit
8c02eb3
·
verified ·
1 Parent(s): 48c487d

Update q_generator1.py

Browse files
Files changed (1) hide show
  1. q_generator1.py +34 -34
q_generator1.py CHANGED
@@ -1,34 +1,34 @@
1
- from transformers import T5Tokenizer, AutoModelForSeq2SeqLM, pipeline
2
-
3
- class QGenerator:
4
- def __init__(self):
5
- tokenizer = T5Tokenizer.from_pretrained("valhalla/t5-small-qg-hl", use_fast=False)
6
- model = AutoModelForSeq2SeqLM.from_pretrained("valhalla/t5-small-qg-hl")
7
- self.qg = pipeline("text2text-generation", model=model, tokenizer=tokenizer)
8
-
9
- def split_sentences(self, text):
10
- # Simple sentence splitting (for better results, use nltk or spacy)
11
- return [s.strip() for s in text.split('.') if s.strip()]
12
-
13
- def chunk_text(self, text, chunk_size=512):
14
- return [text[i:i+chunk_size] for i in range(0, len(text), chunk_size)]
15
-
16
- def generate(self, text, max_questions=5):
17
- questions = []
18
- sentences = self.split_sentences(text)
19
-
20
- for sentence in sentences:
21
- if len(questions) >= max_questions:
22
- break
23
-
24
- input_text = f"generate question: {sentence} </s>"
25
- try:
26
- result = self.qg(input_text, max_length=64, num_return_sequences=1)[0]
27
- question = result["generated_text"]
28
- if question and question not in questions:
29
- questions.append(question)
30
- except Exception as e:
31
- print("Error generating question:", e)
32
- continue
33
-
34
- return questions
 
1
+ from transformers import T5Tokenizer, AutoModelForSeq2SeqLM, pipeline
2
+
3
+ class QGenerator:
4
+ def __init__(self):
5
+ tokenizer = T5Tokenizer.from_pretrained("valhalla/t5-small-qg-hl", use_fast=False)
6
+ model = AutoModelForSeq2SeqLM.from_pretrained("valhalla/t5-small-qg-hl")
7
+ self.qg = pipeline("text2text-generation", model=model, tokenizer=tokenizer)
8
+
9
+ def split_sentences(self, text):
10
+ # Simple sentence splitting (for better results, use nltk or spacy)
11
+ return [s.strip() for s in text.split('.') if s.strip()]
12
+
13
+ def chunk_text(self, text, chunk_size=512):
14
+ return [text[i:i+chunk_size] for i in range(0, len(text), chunk_size)]
15
+
16
+ def generate(self, text, max_questions=1):
17
+ questions = []
18
+ sentences = self.split_sentences(text)
19
+
20
+ for sentence in sentences:
21
+ if len(questions) >= max_questions:
22
+ break
23
+
24
+ input_text = f"generate question: {sentence} </s>"
25
+ try:
26
+ result = self.qg(input_text, max_length=64, num_return_sequences=1)[0]
27
+ question = result["generated_text"]
28
+ if question and question not in questions:
29
+ questions.append(question)
30
+ except Exception as e:
31
+ print("Error generating question:", e)
32
+ continue
33
+
34
+ return questions