Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -19,15 +19,22 @@ from datetime import datetime
|
|
| 19 |
|
| 20 |
PDF_PATH = "2VBMAPP.pdf" # Your pre-loaded PDF file
|
| 21 |
|
| 22 |
-
class
|
| 23 |
def __init__(self):
|
| 24 |
# Load CLIP for computer vision
|
| 25 |
self.clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
|
| 26 |
self.clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
|
| 27 |
|
| 28 |
# Load PDF
|
| 29 |
-
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
self.current_page = 6 # Start at first Tact page (page 7 in PDF, index 6)
|
| 32 |
|
| 33 |
# Session data
|
|
@@ -37,7 +44,6 @@ class ABA Tutor:
|
|
| 37 |
# Expected labels for each page (built from book content)
|
| 38 |
self.page_labels = self.build_label_map()
|
| 39 |
|
| 40 |
-
print(f"✅ Loaded PDF with {self.total_pages} pages")
|
| 41 |
print(f"✅ CLIP model ready")
|
| 42 |
|
| 43 |
def build_label_map(self):
|
|
@@ -54,6 +60,10 @@ class ABA Tutor:
|
|
| 54 |
|
| 55 |
def get_current_image(self):
|
| 56 |
"""Extract current page as PIL Image for CV"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
page = self.doc.load_page(self.current_page)
|
| 58 |
pix = page.get_pixmap(dpi=100)
|
| 59 |
img_data = pix.tobytes("png")
|
|
@@ -62,6 +72,9 @@ class ABA Tutor:
|
|
| 62 |
|
| 63 |
def get_current_image_html(self):
|
| 64 |
"""Get HTML image for display"""
|
|
|
|
|
|
|
|
|
|
| 65 |
page = self.doc.load_page(self.current_page)
|
| 66 |
pix = page.get_pixmap(dpi=100)
|
| 67 |
img_data = pix.tobytes("png")
|
|
@@ -83,17 +96,22 @@ class ABA Tutor:
|
|
| 83 |
candidates = expected + [spoken_word]
|
| 84 |
candidates = list(set(candidates)) # Remove duplicates
|
| 85 |
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
# Check if spoken word matches any expected label
|
| 99 |
spoken_lower = spoken_word.lower().strip()
|
|
@@ -112,10 +130,14 @@ class ABA Tutor:
|
|
| 112 |
|
| 113 |
def speak(self, text):
|
| 114 |
"""Convert text to speech and return audio file"""
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
|
| 120 |
def get_sd_prompt(self):
|
| 121 |
"""Get the SD prompt for current page based on book content"""
|
|
@@ -134,18 +156,18 @@ class ABA Tutor:
|
|
| 134 |
def listen(self):
|
| 135 |
"""Capture and transcribe child's spoken response"""
|
| 136 |
recognizer = sr.Recognizer()
|
| 137 |
-
|
| 138 |
-
|
| 139 |
recognizer.adjust_for_ambient_noise(source, duration=0.5)
|
| 140 |
audio = recognizer.listen(source, timeout=5, phrase_time_limit=3)
|
| 141 |
text = recognizer.recognize_google(audio)
|
| 142 |
return text.lower().strip()
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
|
| 150 |
def run_trial(self):
|
| 151 |
"""Run one complete assessment trial with CV validation"""
|
|
@@ -245,7 +267,7 @@ class ABA Tutor:
|
|
| 245 |
# 2. INITIALIZE TUTOR
|
| 246 |
# ============================================
|
| 247 |
|
| 248 |
-
tutor =
|
| 249 |
|
| 250 |
# ============================================
|
| 251 |
# 3. GRADIO INTERFACE
|
|
|
|
| 19 |
|
| 20 |
PDF_PATH = "2VBMAPP.pdf" # Your pre-loaded PDF file
|
| 21 |
|
| 22 |
+
class ABATutor: # FIXED: No space in class name
|
| 23 |
def __init__(self):
|
| 24 |
# Load CLIP for computer vision
|
| 25 |
self.clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
|
| 26 |
self.clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
|
| 27 |
|
| 28 |
# Load PDF
|
| 29 |
+
try:
|
| 30 |
+
self.doc = fitz.open(PDF_PATH)
|
| 31 |
+
self.total_pages = len(self.doc)
|
| 32 |
+
print(f"✅ Loaded PDF with {self.total_pages} pages")
|
| 33 |
+
except Exception as e:
|
| 34 |
+
print(f"⚠️ Could not load PDF: {e}")
|
| 35 |
+
self.doc = None
|
| 36 |
+
self.total_pages = 1
|
| 37 |
+
|
| 38 |
self.current_page = 6 # Start at first Tact page (page 7 in PDF, index 6)
|
| 39 |
|
| 40 |
# Session data
|
|
|
|
| 44 |
# Expected labels for each page (built from book content)
|
| 45 |
self.page_labels = self.build_label_map()
|
| 46 |
|
|
|
|
| 47 |
print(f"✅ CLIP model ready")
|
| 48 |
|
| 49 |
def build_label_map(self):
|
|
|
|
| 60 |
|
| 61 |
def get_current_image(self):
|
| 62 |
"""Extract current page as PIL Image for CV"""
|
| 63 |
+
if self.doc is None:
|
| 64 |
+
# Return a blank image if PDF not loaded
|
| 65 |
+
return Image.new('RGB', (800, 600), color='white')
|
| 66 |
+
|
| 67 |
page = self.doc.load_page(self.current_page)
|
| 68 |
pix = page.get_pixmap(dpi=100)
|
| 69 |
img_data = pix.tobytes("png")
|
|
|
|
| 72 |
|
| 73 |
def get_current_image_html(self):
|
| 74 |
"""Get HTML image for display"""
|
| 75 |
+
if self.doc is None:
|
| 76 |
+
return '<p style="text-align:center;padding:50px;">⚠️ PDF not loaded. Please ensure 2VBMAPP.pdf is in the Space.</p>'
|
| 77 |
+
|
| 78 |
page = self.doc.load_page(self.current_page)
|
| 79 |
pix = page.get_pixmap(dpi=100)
|
| 80 |
img_data = pix.tobytes("png")
|
|
|
|
| 96 |
candidates = expected + [spoken_word]
|
| 97 |
candidates = list(set(candidates)) # Remove duplicates
|
| 98 |
|
| 99 |
+
try:
|
| 100 |
+
# Run CLIP
|
| 101 |
+
inputs = self.clip_processor(text=candidates, images=image, return_tensors="pt", padding=True)
|
| 102 |
+
with torch.no_grad():
|
| 103 |
+
outputs = self.clip_model(**inputs)
|
| 104 |
+
logits_per_image = outputs.logits_per_image
|
| 105 |
+
probs = logits_per_image.softmax(dim=1).numpy()
|
| 106 |
+
|
| 107 |
+
# Get best match
|
| 108 |
+
best_idx = probs[0].argmax()
|
| 109 |
+
best_label = candidates[best_idx]
|
| 110 |
+
confidence = float(probs[0][best_idx])
|
| 111 |
+
except Exception as e:
|
| 112 |
+
print(f"CLIP error: {e}")
|
| 113 |
+
best_label = spoken_word
|
| 114 |
+
confidence = 0.5
|
| 115 |
|
| 116 |
# Check if spoken word matches any expected label
|
| 117 |
spoken_lower = spoken_word.lower().strip()
|
|
|
|
| 130 |
|
| 131 |
def speak(self, text):
|
| 132 |
"""Convert text to speech and return audio file"""
|
| 133 |
+
try:
|
| 134 |
+
tts = gTTS(text=text, lang="en", slow=False)
|
| 135 |
+
audio_path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3").name
|
| 136 |
+
tts.save(audio_path)
|
| 137 |
+
return audio_path
|
| 138 |
+
except Exception as e:
|
| 139 |
+
print(f"TTS error: {e}")
|
| 140 |
+
return None
|
| 141 |
|
| 142 |
def get_sd_prompt(self):
|
| 143 |
"""Get the SD prompt for current page based on book content"""
|
|
|
|
| 156 |
def listen(self):
|
| 157 |
"""Capture and transcribe child's spoken response"""
|
| 158 |
recognizer = sr.Recognizer()
|
| 159 |
+
try:
|
| 160 |
+
with sr.Microphone() as source:
|
| 161 |
recognizer.adjust_for_ambient_noise(source, duration=0.5)
|
| 162 |
audio = recognizer.listen(source, timeout=5, phrase_time_limit=3)
|
| 163 |
text = recognizer.recognize_google(audio)
|
| 164 |
return text.lower().strip()
|
| 165 |
+
except sr.WaitTimeoutError:
|
| 166 |
+
return "[no response]"
|
| 167 |
+
except sr.UnknownValueError:
|
| 168 |
+
return "[could not understand]"
|
| 169 |
+
except Exception as e:
|
| 170 |
+
return f"[error: {str(e)}]"
|
| 171 |
|
| 172 |
def run_trial(self):
|
| 173 |
"""Run one complete assessment trial with CV validation"""
|
|
|
|
| 267 |
# 2. INITIALIZE TUTOR
|
| 268 |
# ============================================
|
| 269 |
|
| 270 |
+
tutor = ABATutor() # FIXED: Class name without space
|
| 271 |
|
| 272 |
# ============================================
|
| 273 |
# 3. GRADIO INTERFACE
|