iitmbs24f commited on
Commit
479ddc9
·
verified ·
1 Parent(s): 8d6b09c

Upload 16 files

Browse files
Files changed (3) hide show
  1. app/llm.py +12 -1
  2. app/media_processor.py +11 -7
  3. app/solver.py +70 -22
app/llm.py CHANGED
@@ -166,7 +166,7 @@ Respond in JSON format:
166
  return {"raw_response": response}
167
 
168
 
169
- async def solve_with_llm(question: str, available_data: Dict[str, Any]) -> Optional[str]:
170
  """
171
  Use LLM to solve a quiz question.
172
 
@@ -189,6 +189,15 @@ async def solve_with_llm(question: str, available_data: Dict[str, Any]) -> Optio
189
  format_instructions = "\nIMPORTANT: Extract ONLY the git commands. If multiple commands are requested, return them separated by newlines."
190
  elif 'shell command' in question_lower:
191
  format_instructions = "\nIMPORTANT: Extract ONLY the shell commands. Return them exactly as they should be executed."
 
 
 
 
 
 
 
 
 
192
 
193
  prompt = f"""Solve this quiz question:
194
 
@@ -196,11 +205,13 @@ Question: {question}
196
 
197
  Available Data:
198
  {available_data}
 
199
  {format_instructions}
200
 
201
  Provide a clear, concise answer. If the answer should be in JSON format, provide valid JSON.
202
  If it's a calculation, show your work briefly.
203
  If it's a command or path, return ONLY that command or path without any explanation.
 
204
  """
205
 
206
  return await ask_gpt(prompt, max_tokens=3000)
 
166
  return {"raw_response": response}
167
 
168
 
169
+ async def solve_with_llm(question: str, available_data: Dict[str, Any], question_type: Optional[str] = None) -> Optional[str]:
170
  """
171
  Use LLM to solve a quiz question.
172
 
 
189
  format_instructions = "\nIMPORTANT: Extract ONLY the git commands. If multiple commands are requested, return them separated by newlines."
190
  elif 'shell command' in question_lower:
191
  format_instructions = "\nIMPORTANT: Extract ONLY the shell commands. Return them exactly as they should be executed."
192
+ elif 'transcribe' in question_lower or 'passphrase' in question_lower or 'spoken phrase' in question_lower:
193
+ format_instructions = "\nIMPORTANT: This is an audio transcription question. If you cannot access the audio file directly, try to infer the answer from the question context or available data. Return the transcribed phrase with any codes or numbers mentioned."
194
+
195
+ # Check if we have audio transcription data
196
+ audio_data = ""
197
+ if 'audio_transcription' in available_data:
198
+ audio_data = f"\nAudio Transcription: {available_data['audio_transcription']}"
199
+ elif 'audio' in str(available_data).lower():
200
+ audio_data = "\nNote: An audio file is mentioned in the question but transcription is not available. Try to solve based on the question context."
201
 
202
  prompt = f"""Solve this quiz question:
203
 
 
205
 
206
  Available Data:
207
  {available_data}
208
+ {audio_data}
209
  {format_instructions}
210
 
211
  Provide a clear, concise answer. If the answer should be in JSON format, provide valid JSON.
212
  If it's a calculation, show your work briefly.
213
  If it's a command or path, return ONLY that command or path without any explanation.
214
+ If it's an audio transcription, return the spoken phrase with any codes or numbers.
215
  """
216
 
217
  return await ask_gpt(prompt, max_tokens=3000)
app/media_processor.py CHANGED
@@ -88,14 +88,18 @@ Return only the transcribed text, nothing else."""
88
  except Exception as e:
89
  logger.debug(f"OpenAI Whisper not available: {e}")
90
 
91
- # Fallback: Use LLM with description of audio
92
- # This is a workaround - ideally we'd use a proper speech-to-text API
93
- prompt = f"""I have an audio file from this URL: {audio_url}
94
- Please provide a transcription of what might be in this audio file based on the context.
95
- If you cannot transcribe it directly, describe what type of audio it might be and any patterns you can infer."""
96
 
97
- result = await ask_gpt(prompt, max_tokens=1000)
98
- return result
 
 
 
 
99
 
100
  async def process_video_from_url(self, video_url: str) -> Optional[Dict[str, Any]]:
101
  """
 
88
  except Exception as e:
89
  logger.debug(f"OpenAI Whisper not available: {e}")
90
 
91
+ # For now, we can't directly transcribe audio via OpenRouter
92
+ # But we can try to download and analyze the audio file
93
+ # For passphrase quizzes, we need the actual transcription
94
+ # Try to use a vision-capable model that might support audio
95
+ # Or return a placeholder that indicates we need transcription
96
 
97
+ # Since we can't actually transcribe, return None and let the system
98
+ # use LLM to solve based on the question context
99
+ logger.warning(f"Cannot transcribe audio directly - audio transcription requires specialized API")
100
+
101
+ # Return None - the system will fall back to LLM solving
102
+ return None
103
 
104
  async def process_video_from_url(self, video_url: str) -> Optional[Dict[str, Any]]:
105
  """
app/solver.py CHANGED
@@ -157,8 +157,20 @@ class QuizSolver:
157
  command_match = re.search(r'(uv\s+http\s+get\s+[^\n<>"]+(?:\s+-H\s+"[^"]+")?)', reason, re.IGNORECASE)
158
  if command_match:
159
  correct_command = command_match.group(1).strip()
 
160
  if email:
161
- correct_command = correct_command.replace('<your email>', email).replace('<email>', email)
 
 
 
 
 
 
 
 
 
 
 
162
  logger.info(f"Retrying with correct command: {correct_command[:100]}...")
163
  # Retry submission with correct command
164
  retry_response = await self._submit_answer(
@@ -166,6 +178,9 @@ class QuizSolver:
166
  )
167
  if isinstance(retry_response, dict) and retry_response.get('correct'):
168
  response = retry_response
 
 
 
169
  elif 'git add' in reason.lower() and 'git commit' in reason.lower():
170
  # Extract git commands from reason
171
  need_match = re.search(r'[Nn]eed\s+(git\s+add\s+[^\s]+)\s+then\s+(git\s+commit\s+[^\n<>"]+)', reason, re.IGNORECASE)
@@ -277,19 +292,29 @@ class QuizSolver:
277
  for audio_url in media_files['audio']:
278
  try:
279
  remaining = self._check_time_remaining()
280
- if remaining >= 20.0: # Need more time to process audio
 
 
 
 
281
  transcription = await media_processor.process_audio_from_url(audio_url)
282
  if transcription:
283
  # Use transcription to solve
284
  available_data['audio_transcription'] = transcription
285
  # For passphrase quizzes, return the transcription directly
286
- if 'transcribe' in question.lower() or 'passphrase' in question.lower():
287
  logger.info(f"Returning audio transcription as answer: {transcription[:100]}...")
288
  return transcription
289
  # Try to extract answer from transcription
290
  answer = self._extract_answer_from_transcription(transcription, question)
291
  if answer:
292
  return answer
 
 
 
 
 
 
293
  except Exception as e:
294
  logger.warning(f"Error processing audio {audio_url}: {e}")
295
  continue # Try next audio file
@@ -439,12 +464,25 @@ class QuizSolver:
439
 
440
  # Strategy 7: Use LLM to solve (only if we have enough time)
441
  remaining = self._check_time_remaining()
 
 
 
 
442
  # Only use LLM if we have enough time AND haven't found answer yet
443
  # Reserve at least 10s for submission
444
- if remaining >= 25.0: # Increased threshold to ensure time for submission
445
  logger.info("Attempting to solve with LLM...")
446
  try:
447
- llm_answer = await solve_with_llm(question, available_data)
 
 
 
 
 
 
 
 
 
448
  if llm_answer:
449
  # Try to parse as JSON if it looks like JSON
450
  json_answer = extract_json_from_text(llm_answer)
@@ -456,7 +494,7 @@ class QuizSolver:
456
  # Try to extract any useful information from the error
457
  pass
458
  else:
459
- logger.warning(f"Skipping LLM call - insufficient time remaining ({remaining:.1f}s, need 25s)")
460
 
461
  # Strategy 8: Fallback - try to extract a simple answer from the question
462
  # Many quiz pages have the answer in the question itself
@@ -1212,23 +1250,33 @@ class QuizSolver:
1212
  question_lower = question.lower()
1213
 
1214
  # Check if it's a math expression
 
1215
  if any(op in question for op in ['+', '-', '*', '/', '=', 'sqrt', 'sin', 'cos', 'tan']):
1216
- # Try to extract and solve math expression
1217
- # Look for expressions like "2+2", "10*5", etc.
1218
- expr_patterns = [
1219
- r'(\d+\s*[+\-*/]\s*\d+)',
1220
- r'([\d+\-*/()\s]+)',
1221
- r'calculate\s+([\d+\-*/()\s]+)',
1222
- r'what\s+is\s+([\d+\-*/()\s]+)',
1223
- ]
1224
-
1225
- for pattern in expr_patterns:
1226
- match = re.search(pattern, question)
1227
- if match:
1228
- expr = match.group(1).strip()
1229
- result = calc_engine.solve_math_expression(expr)
1230
- if result is not None:
1231
- return int(result) if abs(result - int(result)) < 0.0001 else result
 
 
 
 
 
 
 
 
 
1232
 
1233
  # Check for sum of numbers in text
1234
  if 'sum' in question_lower or 'total' in question_lower or 'add' in question_lower:
 
157
  command_match = re.search(r'(uv\s+http\s+get\s+[^\n<>"]+(?:\s+-H\s+"[^"]+")?)', reason, re.IGNORECASE)
158
  if command_match:
159
  correct_command = command_match.group(1).strip()
160
+ # Substitute email - handle all possible formats
161
  if email:
162
+ correct_command = correct_command.replace('<your email>', email)
163
+ correct_command = correct_command.replace('<email>', email)
164
+ # Replace any placeholder email addresses using regex
165
+ correct_command = re.sub(r'email=user@example\.com', f'email={email}', correct_command, flags=re.IGNORECASE)
166
+ correct_command = re.sub(r'email="user@example\.com"', f'email={email}', correct_command, flags=re.IGNORECASE)
167
+ # Also handle if email parameter is missing entirely
168
+ if 'email=' not in correct_command and '?' in correct_command:
169
+ correct_command = correct_command.replace('?', f'?email={email}&') if '&' not in correct_command.split('?')[1] else correct_command.replace('?', f'?email={email}&')
170
+ elif 'email=' not in correct_command:
171
+ # Add email parameter
172
+ separator = '&' if '?' in correct_command else '?'
173
+ correct_command = f"{correct_command}{separator}email={email}"
174
  logger.info(f"Retrying with correct command: {correct_command[:100]}...")
175
  # Retry submission with correct command
176
  retry_response = await self._submit_answer(
 
178
  )
179
  if isinstance(retry_response, dict) and retry_response.get('correct'):
180
  response = retry_response
181
+ logger.info("Retry successful!")
182
+ else:
183
+ logger.warning(f"Retry still failed: {retry_response.get('reason', 'Unknown error')}")
184
  elif 'git add' in reason.lower() and 'git commit' in reason.lower():
185
  # Extract git commands from reason
186
  need_match = re.search(r'[Nn]eed\s+(git\s+add\s+[^\s]+)\s+then\s+(git\s+commit\s+[^\n<>"]+)', reason, re.IGNORECASE)
 
292
  for audio_url in media_files['audio']:
293
  try:
294
  remaining = self._check_time_remaining()
295
+ # Process audio - it's critical for passphrase quizzes
296
+ # Reduced threshold to allow processing even with limited time
297
+ remaining = self._check_time_remaining()
298
+ if remaining >= 5.0: # Very low threshold - process if we have any reasonable time
299
+ logger.info(f"Processing audio file: {audio_url}")
300
  transcription = await media_processor.process_audio_from_url(audio_url)
301
  if transcription:
302
  # Use transcription to solve
303
  available_data['audio_transcription'] = transcription
304
  # For passphrase quizzes, return the transcription directly
305
+ if 'transcribe' in question.lower() or 'passphrase' in question.lower() or 'spoken phrase' in question.lower():
306
  logger.info(f"Returning audio transcription as answer: {transcription[:100]}...")
307
  return transcription
308
  # Try to extract answer from transcription
309
  answer = self._extract_answer_from_transcription(transcription, question)
310
  if answer:
311
  return answer
312
+ else:
313
+ # If transcription failed, use LLM to solve based on question
314
+ # The LLM might be able to infer or we can try other strategies
315
+ logger.info("Audio transcription unavailable, will use LLM to solve")
316
+ else:
317
+ logger.warning(f"Skipping audio processing - insufficient time ({remaining:.1f}s remaining)")
318
  except Exception as e:
319
  logger.warning(f"Error processing audio {audio_url}: {e}")
320
  continue # Try next audio file
 
464
 
465
  # Strategy 7: Use LLM to solve (only if we have enough time)
466
  remaining = self._check_time_remaining()
467
+ # For audio passphrase questions, use LLM even with less time
468
+ is_audio_question = 'transcribe' in question.lower() or 'passphrase' in question.lower() or 'spoken phrase' in question.lower()
469
+ min_time_needed = 15.0 if is_audio_question else 25.0 # Lower threshold for audio questions
470
+
471
  # Only use LLM if we have enough time AND haven't found answer yet
472
  # Reserve at least 10s for submission
473
+ if remaining >= min_time_needed:
474
  logger.info("Attempting to solve with LLM...")
475
  try:
476
+ # Determine question type for better LLM handling
477
+ question_type = None
478
+ if 'transcribe' in question.lower() or 'passphrase' in question.lower():
479
+ question_type = 'audio'
480
+ elif 'command string' in question.lower():
481
+ question_type = 'command'
482
+ elif 'git' in question.lower():
483
+ question_type = 'git'
484
+
485
+ llm_answer = await solve_with_llm(question, available_data, question_type)
486
  if llm_answer:
487
  # Try to parse as JSON if it looks like JSON
488
  json_answer = extract_json_from_text(llm_answer)
 
494
  # Try to extract any useful information from the error
495
  pass
496
  else:
497
+ logger.warning(f"Skipping LLM call - insufficient time remaining ({remaining:.1f}s, need {min_time_needed}s)")
498
 
499
  # Strategy 8: Fallback - try to extract a simple answer from the question
500
  # Many quiz pages have the answer in the question itself
 
1250
  question_lower = question.lower()
1251
 
1252
  # Check if it's a math expression
1253
+ # Don't treat paths like /project2-uv as math expressions
1254
  if any(op in question for op in ['+', '-', '*', '/', '=', 'sqrt', 'sin', 'cos', 'tan']):
1255
+ # Skip if it looks like a URL or path (contains http, /, or .)
1256
+ if 'http' in question or question.startswith('/') or '.' in question.split()[0] if question.split() else False:
1257
+ pass # Skip math processing for URLs/paths
1258
+ else:
1259
+ # Try to extract and solve math expression
1260
+ # Look for expressions like "2+2", "10*5", etc.
1261
+ expr_patterns = [
1262
+ r'(\d+\s*[+\-*/]\s*\d+)', # Simple: "2+2"
1263
+ r'calculate\s+([\d+\-*/()\s]+)', # "calculate 2+2"
1264
+ r'what\s+is\s+([\d+\-*/()\s]+)', # "what is 2+2"
1265
+ ]
1266
+
1267
+ for pattern in expr_patterns:
1268
+ match = re.search(pattern, question)
1269
+ if match:
1270
+ expr = match.group(1).strip()
1271
+ # Validate it's actually a math expression (has numbers and operators)
1272
+ if re.search(r'\d+.*[+\-*/]', expr) or re.search(r'[+\-*/].*\d+', expr):
1273
+ try:
1274
+ result = calc_engine.solve_math_expression(expr)
1275
+ if result is not None:
1276
+ return int(result) if abs(result - int(result)) < 0.0001 else result
1277
+ except Exception as e:
1278
+ logger.debug(f"Math expression evaluation failed (not a real math problem): {e}")
1279
+ pass # Not a real math expression, continue
1280
 
1281
  # Check for sum of numbers in text
1282
  if 'sum' in question_lower or 'total' in question_lower or 'add' in question_lower: