long response handling

config/default_responses.py

@@ -1,16 +0,0 @@
-# config/default_responses.py
-
-default_help_response = (
-    "🤖 I'm a Topcoder MCP Agent! I can:\n"
-    "- Show active challenges\n"
-    "- Fetch member profiles (with handle)\n"
-    "- Answer questions about Topcoder processes\n\n"
-    "I won't be able to help with general knowledge or personal tasks."
-)
-
-help_triggers = [
-    "what can you do?",
-    "what can you do for me",
-    "help",
-    "how can you help me?",
-] 

config/long_fields_config.json

@@ -0,0 +1,16 @@
+{
+  "_note": "This file is human- and LLM-editable. It tracks long field paths per tool/resource.",
+  "tools": {
+    "query-tc-skills": [
+      "description"
+    ],
+    "query-tc-challenges": [
+      "description"
+    ]
+  },
+  "resources": {
+    "Member_V6_API_Swagger": [
+      "skills"
+    ]
+  }
+}

config/prompt_templates.py

@@ -87,22 +87,20 @@ Valid parameter names to extract:
 Respond ONLY with a JSON object containing the extracted parameters:
 {{"params": {{"param1": "value1", "param2": "value2"}} }}"""
 
-    GRADIO_RESPONSE_SUMMARIZATION = """You are a Topcoder assistant. Provide accurate information based on the actual data.
+    GRADIO_RESPONSE_SUMMARIZATION = """You are a Topcoder assistant. Provide clear, helpful responses based on the actual data.
 
-The user asked: "{user_message}"
+USER QUERY: {user_message}
 {context}
 
-Response data:
+RESPONSE DATA:
 {filtered_data}
 
-Instructions:
-1. If the data shows results exist, say so with the actual numbers
-2. If the data shows no results, say "No results found"
-3. Don't make up information - only use what's in the data
-4. If there are multiple items, mention the total count and show a few examples
-5. Be specific about what was found vs what was requested
-
-Write a clear, helpful response based on the actual data provided."""
+INSTRUCTIONS:
+- Give a natural, conversational response
+- If results exist, summarize what was found
+- If no results, say "No results found"
+- If some fields were omitted due to length, simply ask: "Would you like to see more details about [field names]?"
+- Keep it simple and helpful - don't expose technical details"""
 
     # Tool decision prompt template
     TOOL_DECISION_PROMPT = """You are an assistant that ONLY works with Topcoder's Member Communication Platform (MCP).

config/settings.py

@@ -30,4 +30,10 @@ EXCLUDE_AUTH_REQUIRED_ENDPOINTS = os.getenv("EXCLUDE_AUTH_REQUIRED_ENDPOINTS", "
 EXCLUDE_401_RESPONSES = os.getenv("EXCLUDE_401_RESPONSES", "true").lower() == "true"
 
 # Set to True to be more strict and exclude endpoints that mention authentication in descriptions
-STRICT_AUTH_FILTERING = os.getenv("STRICT_AUTH_FILTERING", "false").lower() == "true"
+STRICT_AUTH_FILTERING = os.getenv("STRICT_AUTH_FILTERING", "false").lower() == "true"
+
+# Long-field detection thresholds 
+# Tune these to control when fields are treated as long and omitted by default
+LONG_FIELD_STRING_MAX_CHARS = int(os.getenv("LONG_FIELD_STRING_MAX_CHARS", "500"))
+LONG_FIELD_ARRAY_MAX_ITEMS = int(os.getenv("LONG_FIELD_ARRAY_MAX_ITEMS", "25"))
+LONG_FIELD_ARRAY_MAX_BYTES = int(os.getenv("LONG_FIELD_ARRAY_MAX_BYTES", "10000"))

scripts/demo_conversation_history.py

@@ -1,97 +0,0 @@
-#!/usr/bin/env python3
-"""
-Demo script for conversation history functionality.
-This script demonstrates how the conversation history system works.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-# Add src to path so we can import modules
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-
-from src.domain.services.conversation_history import ConversationHistoryService
-from src.domain.services.session_manager import SessionManager
-
-async def demo_conversation_history():
-    """Demonstrate conversation history functionality."""
-    
-    print("🧠 Conversation History Demo")
-    print("=" * 50)
-    
-    # Initialize services
-    history_service = ConversationHistoryService(default_max_history=10, default_max_tokens=2000)
-    session_manager = SessionManager(history_service)
-    
-    # Create a session
-    session_id = session_manager.create_session()
-    print(f"📝 Created session: {session_id}")
-    
-    # Simulate a conversation about challenges
-    conversation = [
-        ("user", "Show me active challenges"),
-        ("assistant", "I found 15 active challenges. Here are some highlights: AI Challenge #123, Web Development #456, and Mobile App #789."),
-        ("user", "What about AI challenges specifically?"),
-        ("assistant", "Looking at the active challenges, here are the AI-specific ones: AI Challenge #123 (Machine Learning), AI Challenge #124 (Computer Vision), and AI Challenge #125 (NLP)."),
-        ("user", "Tell me more about the Machine Learning challenge"),
-        ("assistant", "AI Challenge #123 (Machine Learning) has a prize pool of $10,000, deadline in 2 weeks, and focuses on developing an image classification model."),
-        ("user", "What skills are needed for this challenge?"),
-        ("assistant", "For AI Challenge #123, you'll need: Python, TensorFlow/PyTorch, Computer Vision, Machine Learning, and experience with image classification algorithms."),
-        ("user", "Are there any similar challenges?"),
-        ("assistant", "Yes, there are similar challenges: AI Challenge #124 (Computer Vision) and AI Challenge #125 (NLP) both require similar AI/ML skills but focus on different domains.")
-    ]
-    
-    print("\n💬 Simulating conversation...")
-    
-    # Add messages to history
-    for i, (role, content) in enumerate(conversation):
-        history_service.add_message(session_id, role, content)
-        print(f"  {role.upper()}: {content[:50]}{'...' if len(content) > 50 else ''}")
-        
-        # Show history after each message
-        if i % 2 == 1:  # After each exchange
-            current_history = history_service.get_conversation_history(session_id)
-            print(f"  📚 History length: {len(current_history)} messages")
-    
-    # Show final history
-    print("\n📋 Final Conversation History:")
-    final_history = history_service.get_conversation_history(session_id)
-    for i, msg in enumerate(final_history):
-        role_icon = "👤" if msg["role"] == "user" else "🤖"
-        print(f"  {role_icon} {msg['role'].upper()}: {msg['content'][:60]}{'...' if len(msg['content']) > 60 else ''}")
-    
-    # Show session info
-    print("\n📊 Session Information:")
-    session_info = history_service.get_session_info(session_id)
-    for key, value in session_info.items():
-        if key != "session_id":  # Skip the long session ID
-            print(f"  {key}: {value}")
-    
-    # Show recent messages
-    print("\n🕒 Recent Messages (last 3):")
-    recent = history_service.get_recent_messages(session_id, count=3)
-    for msg in recent:
-        role_icon = "👤" if msg["role"] == "user" else "🤖"
-        print(f"  {role_icon} {msg['role'].upper()}: {msg['content'][:50]}{'...' if len(msg['content']) > 50 else ''}")
-    
-    # Test history limits
-    print("\n🧪 Testing History Limits:")
-    print("  Adding 15 more messages to test sliding window...")
-    
-    for i in range(15):
-        history_service.add_message(session_id, "user", f"Test message {i}")
-        history_service.add_message(session_id, "assistant", f"Test response {i}")
-    
-    final_history = history_service.get_conversation_history(session_id)
-    print(f"  Final history length: {len(final_history)} (should be <= 10)")
-    print(f"  ✅ History limit working: {len(final_history) <= 10}")
-    
-    # Cleanup
-    session_manager.remove_session(session_id)
-    print(f"\n🧹 Cleaned up session: {session_id}")
-    
-    print("\n✅ Demo completed successfully!")
-
-if __name__ == "__main__":
-    asyncio.run(demo_conversation_history()) 

scripts/diagnose_issues.py

@@ -1,189 +0,0 @@
-#!/usr/bin/env python3
-"""
-Diagnostic script to identify issues with tool selection and execution.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-# Add src to path so we can import modules
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-
-from src.infrastructure.providers.llm_provider import LLMClient
-from src.application.services.prompt_service import PromptBuilder
-from src.application.services.tool_service import ToolExecutor
-from src.mcp.compact_utils import CompactSpecsUtils
-from src.mcp.client.transport import load_server_config, StreamableHttpMCPClient
-
-async def diagnose_components():
-    """Diagnose each component of the system."""
-    
-    print("🔍 System Diagnostics")
-    print("=" * 50)
-    
-    # 1. Check compact specs
-    print("\n1. 📋 Checking Compact Specs...")
-    compact_utils = CompactSpecsUtils()
-    
-    working_tools = compact_utils.get_working_tools()
-    working_resources = compact_utils.get_working_resources()
-    
-    print(f"   ✅ Working tools: {working_tools}")
-    print(f"   ✅ Working resources: {working_resources}")
-    
-    if not working_tools and not working_resources:
-        print("   ❌ No tools or resources found!")
-        return False
-    
-    # 2. Check LLM provider
-    print("\n2. 🤖 Checking LLM Provider...")
-    try:
-        llm_client = LLMClient()
-        print("   ✅ LLM client initialized")
-        
-        # Test simple chat
-        response = await llm_client.chat([{"role": "user", "content": "Hello"}])
-        print(f"   ✅ LLM chat working: {len(response)} characters")
-        
-    except Exception as e:
-        print(f"   ❌ LLM provider error: {e}")
-        return False
-    
-    # 3. Check prompt builder
-    print("\n3. 📝 Checking Prompt Builder...")
-    try:
-        prompt_builder = PromptBuilder()
-        prompt = prompt_builder.build_tool_decision_prompt("Show me challenges")
-        print(f"   ✅ Prompt builder working: {len(prompt)} characters")
-        
-    except Exception as e:
-        print(f"   ❌ Prompt builder error: {e}")
-        return False
-    
-    # 4. Check tool executor
-    print("\n4. ⚡ Checking Tool Executor...")
-    try:
-        tool_executor = ToolExecutor()
-        available_tools = tool_executor.get_available_tools()
-        available_resources = tool_executor.get_available_resources()
-        
-        print(f"   ✅ Available tools: {available_tools}")
-        print(f"   ✅ Available resources: {available_resources}")
-        
-    except Exception as e:
-        print(f"   ❌ Tool executor error: {e}")
-        return False
-    
-    # 5. Check MCP connection
-    print("\n5. 🔌 Checking MCP Connection...")
-    try:
-        server_config = load_server_config()
-        print(f"   ✅ Server config loaded: {server_config}")
-        
-        async with StreamableHttpMCPClient(server_config) as client:
-            init_data = await client.initialize()
-            print(f"   ✅ MCP connection successful: {init_data}")
-            
-            # Test tools/list
-            tools_data = await client.call("tools/list", {})
-            tools = tools_data.get("result", {}).get("tools", [])
-            print(f"   ✅ Tools available: {len(tools)} tools")
-            
-    except Exception as e:
-        print(f"   ❌ MCP connection error: {e}")
-        return False
-    
-    print("\n✅ All components working!")
-    return True
-
-async def test_specific_issue():
-    """Test a specific user query to identify the issue."""
-    
-    print("\n🎯 Testing Specific Query")
-    print("=" * 50)
-    
-    user_message = "Show me active challenges"
-    print(f"User message: {user_message}")
-    
-    # Initialize services
-    llm_client = LLMClient()
-    prompt_builder = PromptBuilder()
-    tool_executor = ToolExecutor()
-    
-    # Step 1: Tool decision
-    print("\nStep 1: Tool Decision")
-    decision_prompt = prompt_builder.build_tool_decision_prompt(user_message)
-    print(f"Prompt length: {len(decision_prompt)} characters")
-    
-    try:
-        decision_json = await llm_client.decide_tool(decision_prompt)
-        print(f"Decision: {decision_json}")
-        
-        tool = decision_json.get("tool")
-        resource = decision_json.get("resource")
-        params = decision_json.get("params", {})
-        
-        print(f"Tool: {tool}")
-        print(f"Resource: {resource}")
-        print(f"Params: {params}")
-        
-    except Exception as e:
-        print(f"❌ Tool decision failed: {e}")
-        return
-    
-    # Step 2: Tool execution
-    if tool or resource:
-        print("\nStep 2: Tool Execution")
-        
-        available_tools = tool_executor.get_available_tools()
-        available_resources = tool_executor.get_available_resources()
-        
-        if tool and tool in available_tools:
-            target_name = tool
-            target_type = "tool"
-        elif resource and resource in available_resources:
-            target_name = resource
-            target_type = "resource"
-        else:
-            print(f"❌ Invalid target: {tool or resource}")
-            return
-        
-        print(f"Executing: {target_name} ({target_type})")
-        
-        from src.domain.models.tool import ToolRequest
-        request = ToolRequest(tool=target_name, params=params)
-        
-        try:
-            result = await tool_executor.execute(request)
-            print(f"Result status: {result.status}")
-            
-            if result.status == "success":
-                print("✅ Tool execution successful!")
-                if result.data:
-                    print(f"Data length: {len(str(result.data))} characters")
-            else:
-                print(f"❌ Tool execution failed: {result.message}")
-                
-        except Exception as e:
-            print(f"❌ Exception during execution: {e}")
-            import traceback
-            traceback.print_exc()
-
-async def main():
-    """Run diagnostics."""
-    try:
-        # Run component diagnostics
-        if await diagnose_components():
-            # If all components are working, test specific issue
-            await test_specific_issue()
-        else:
-            print("\n❌ Component diagnostics failed. Please check the issues above.")
-            
-    except Exception as e:
-        print(f"\n❌ Diagnostic failed: {e}")
-        import traceback
-        traceback.print_exc()
-
-if __name__ == "__main__":
-    asyncio.run(main()) 

scripts/migrate_to_v2.py

@@ -1,269 +0,0 @@
-#!/usr/bin/env python3
-"""
-Migration script from v1.x to v2.0 architecture.
-Helps preserve existing data and configuration while upgrading to new structure.
-"""
-
-import os
-import json
-import shutil
-from pathlib import Path
-from typing import Dict, Any, List
-import argparse
-
-
-class MigrationTool:
-    """Tool to migrate from v1.x to v2.0 architecture."""
-    
-    def __init__(self, old_app_dir: str = "app", backup_dir: str = "backup_v1"):
-        self.old_app_dir = Path(old_app_dir)
-        self.backup_dir = Path(backup_dir)
-        self.src_dir = Path("src")
-        
-    def create_backup(self) -> None:
-        """Create backup of old structure."""
-        if self.old_app_dir.exists():
-            print(f"📦 Creating backup in {self.backup_dir}/")
-            if self.backup_dir.exists():
-                shutil.rmtree(self.backup_dir)
-            shutil.copytree(self.old_app_dir, self.backup_dir)
-            print("✅ Backup created successfully")
-        
-    def migrate_config(self) -> None:
-        """Migrate configuration files."""
-        print("🔧 Migrating configuration...")
-        
-        # Migrate .env file format
-        env_file = Path(".env")
-        if env_file.exists():
-            self._migrate_env_file(env_file)
-        
-        # Migrate mcp-config.json if it exists
-        mcp_config = Path("mcp-config.json")
-        if mcp_config.exists():
-            self._migrate_mcp_config(mcp_config)
-            
-        print("✅ Configuration migrated")
-    
-    def _migrate_env_file(self, env_file: Path) -> None:
-        """Migrate .env file to new format."""
-        old_env = env_file.read_text()
-        new_env_lines = []
-        
-        mapping = {
-            "HF_TOKEN": "LLM_API_KEY",
-            "HF_MODEL": "LLM_MODEL", 
-            "LLM_API_URL": "LLM_API_URL",
-            "LLM_TEMPERATURE": "LLM_TEMPERATURE",
-            "MCP_SERVER_URL": "MCP_SERVER_URL",
-            "MCP_TOKEN": "MCP_API_KEY",
-            "EXCLUDE_AUTH_REQUIRED_ENDPOINTS": "EXCLUDE_AUTH_ENDPOINTS",
-        }
-        
-        for line in old_env.split('\n'):
-            if '=' in line and not line.strip().startswith('#'):
-                key, value = line.split('=', 1)
-                key = key.strip()
-                if key in mapping:
-                    new_key = mapping[key]
-                    new_env_lines.append(f"{new_key}={value}")
-                else:
-                    new_env_lines.append(line)
-            else:
-                new_env_lines.append(line)
-        
-        # Add new default values
-        new_defaults = [
-            "",
-            "# New v2.0 settings",
-            "MCP_PROTOCOL_VERSION=2024-11-05",
-            "MCP_CLIENT_NAME=topcoder-mcp-agent",
-            "MCP_CLIENT_VERSION=2.0.0",
-            "LOG_LEVEL=INFO",
-            "DEBUG=false",
-            "ENVIRONMENT=development"
-        ]
-        
-        new_env_content = '\n'.join(new_env_lines + new_defaults)
-        env_file.write_text(new_env_content)
-        print(f"  📝 Updated .env file with new format")
-    
-    def _migrate_mcp_config(self, mcp_config: Path) -> None:
-        """Archive old MCP config file."""
-        backup_config = self.backup_dir / "mcp-config.json"
-        shutil.copy2(mcp_config, backup_config)
-        print(f"  📋 Backed up mcp-config.json")
-    
-    def migrate_data(self) -> None:
-        """Migrate data files."""
-        print("📊 Migrating data files...")
-        
-        # Create new data structure
-        data_dir = Path("data")
-        data_dir.mkdir(exist_ok=True)
-        (data_dir / "cache").mkdir(exist_ok=True)
-        (data_dir / "catalog").mkdir(exist_ok=True)
-        
-        # Migrate mcp_catalog if it exists
-        old_catalog = Path("mcp_catalog")
-        if old_catalog.exists():
-            new_catalog = data_dir / "catalog"
-            if new_catalog.exists():
-                shutil.rmtree(new_catalog)
-            shutil.copytree(old_catalog, new_catalog)
-            print(f"  📁 Migrated mcp_catalog to {new_catalog}")
-        
-        # Migrate any compact specs
-        old_compact_specs = self.old_app_dir / "config" / "compact_api_specs.py"
-        if old_compact_specs.exists():
-            new_specs_dir = data_dir / "specs"
-            new_specs_dir.mkdir(exist_ok=True)
-            
-            # Convert Python file to JSON
-            self._convert_compact_specs_to_json(old_compact_specs, new_specs_dir / "compact_specs.json")
-            
-        print("✅ Data migration completed")
-    
-    def _convert_compact_specs_to_json(self, old_file: Path, new_file: Path) -> None:
-        """Convert old Python compact specs to JSON format."""
-        try:
-            # Read the old file and extract COMPACT_API_SPECS
-            old_content = old_file.read_text()
-            
-            # This is a simplified conversion - in practice, you might want to 
-            # import and execute the Python file to get the actual dictionary
-            if "COMPACT_API_SPECS = {" in old_content:
-                # Extract the dictionary part (this is a simplified approach)
-                start = old_content.find("COMPACT_API_SPECS = {")
-                if start != -1:
-                    # For now, just copy the file for manual migration
-                    shutil.copy2(old_file, new_file.with_suffix('.py.bak'))
-                    print(f"  📄 Backed up compact specs to {new_file.with_suffix('.py.bak')}")
-                    print(f"  ⚠️  Manual migration needed for compact specs format")
-        except Exception as e:
-            print(f"  ⚠️  Could not convert compact specs: {e}")
-    
-    def remove_old_structure(self) -> None:
-        """Remove old app directory structure."""
-        print("🗑️  Removing old structure...")
-        
-        if self.old_app_dir.exists():
-            # Double-check backup exists
-            if not self.backup_dir.exists():
-                print("❌ Backup not found! Skipping removal for safety.")
-                return
-            
-            shutil.rmtree(self.old_app_dir)
-            print(f"✅ Removed old {self.old_app_dir} directory")
-        
-        # Remove old files
-        old_files = ["setup.py"]
-        for file_name in old_files:
-            file_path = Path(file_name)
-            if file_path.exists():
-                file_path.unlink()
-                print(f"  🗑️  Removed {file_name}")
-    
-    def verify_migration(self) -> bool:
-        """Verify migration was successful."""
-        print("🔍 Verifying migration...")
-        
-        required_dirs = [
-            "src",
-            "src/presentation",
-            "src/application", 
-            "src/domain",
-            "src/mcp",
-            "src/infrastructure"
-        ]
-        
-        required_files = [
-            "pyproject.toml",
-            "README.md",
-            "src/main.py"
-        ]
-        
-        missing = []
-        
-        for dir_path in required_dirs:
-            if not Path(dir_path).exists():
-                missing.append(dir_path)
-        
-        for file_path in required_files:
-            if not Path(file_path).exists():
-                missing.append(file_path)
-        
-        if missing:
-            print(f"❌ Missing required items: {missing}")
-            return False
-        
-        print("✅ Migration verification passed")
-        return True
-    
-    def run_migration(self, skip_backup: bool = False, skip_removal: bool = False) -> None:
-        """Run the complete migration process."""
-        print("🚀 Starting migration from v1.x to v2.0...")
-        print()
-        
-        try:
-            if not skip_backup:
-                self.create_backup()
-                print()
-            
-            self.migrate_config()
-            print()
-            
-            self.migrate_data()
-            print()
-            
-            if self.verify_migration():
-                print()
-                if not skip_removal:
-                    self.remove_old_structure()
-                    print()
-                
-                print("🎉 Migration completed successfully!")
-                print()
-                print("Next steps:")
-                print("1. Review your .env file and update any API keys")
-                print("2. Install new dependencies: pip install -e .")
-                print("3. Run the application: python src/main.py")
-                print("4. Check the backup_v1/ directory if you need any old files")
-            else:
-                print("❌ Migration verification failed")
-                
-        except Exception as e:
-            print(f"💥 Migration failed: {e}")
-            print("Your original files are safely backed up in backup_v1/")
-
-
-def main():
-    """Main CLI entry point."""
-    parser = argparse.ArgumentParser(description="Migrate Topcoder MCP Agent to v2.0")
-    parser.add_argument(
-        "--skip-backup", 
-        action="store_true", 
-        help="Skip creating backup of old structure"
-    )
-    parser.add_argument(
-        "--skip-removal", 
-        action="store_true", 
-        help="Skip removing old structure after migration"
-    )
-    parser.add_argument(
-        "--old-app-dir",
-        default="app",
-        help="Path to old app directory (default: app)"
-    )
-    
-    args = parser.parse_args()
-    
-    migrator = MigrationTool(old_app_dir=args.old_app_dir)
-    migrator.run_migration(
-        skip_backup=args.skip_backup,
-        skip_removal=args.skip_removal
-    )
-
-
-if __name__ == "__main__":
-    main()

scripts/test_context_aware_selection.py

@@ -1,207 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for context-aware tool selection with conversation history.
-This script demonstrates how the enhanced tool selection works with conversation context.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-# Add src to path so we can import modules
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-
-# Add root directory to path so we can import config module
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.infrastructure.providers.llm_provider import LLMClient
-from src.application.services.prompt_service import PromptBuilder
-from src.domain.services.conversation_history import ConversationHistoryService
-from config import settings
-
-async def test_context_aware_selection():
-    """Test context-aware tool selection with conversation history."""
-    
-    print("🧠 Testing Context-Aware Tool Selection")
-    print("=" * 50)
-    
-    # Initialize services
-    llm_client = LLMClient()
-    prompt_builder = PromptBuilder()
-    history_service = ConversationHistoryService()
-    session_id = "context_test"
-    
-    # Test conversation scenarios
-    test_scenarios = [
-        {
-            "name": "Challenge Follow-up with AI Filter",
-            "conversation": [
-                ("user", "Show me active challenges"),
-                ("assistant", "I found several active challenges. Here are some highlights: AI Challenge #123, Web Development #456, Mobile App #789."),
-                ("user", "What about AI challenges specifically?")
-            ],
-            "expected_behavior": "Should select query-tc-challenges with track=AI filter"
-        },
-        {
-            "name": "Skills Follow-up with Programming Filter",
-            "conversation": [
-                ("user", "What programming skills are available?"),
-                ("assistant", "I found several programming skills: Python, JavaScript, Java, React, Angular, Node.js."),
-                ("user", "Show me more programming skills")
-            ],
-            "expected_behavior": "Should select query-tc-skills with name filter for programming"
-        },
-        {
-            "name": "Challenge Status Refinement",
-            "conversation": [
-                ("user", "Show me active challenges"),
-                ("assistant", "I found 15 active challenges. Here are some highlights: AI Challenge #123, Web Development #456."),
-                ("user", "What about completed challenges?")
-            ],
-            "expected_behavior": "Should select query-tc-challenges with status=Completed"
-        },
-        {
-            "name": "Skills Technology Refinement",
-            "conversation": [
-                ("user", "What skills are available?"),
-                ("assistant", "I found various skills: Python, JavaScript, Machine Learning, AI, React, Angular."),
-                ("user", "Tell me about AI and Machine Learning skills")
-            ],
-            "expected_behavior": "Should select query-tc-skills with name filter for AI/ML"
-        }
-    ]
-    
-    for i, scenario in enumerate(test_scenarios):
-        print(f"\n📝 Test {i+1}: {scenario['name']}")
-        print(f"Expected behavior: {scenario['expected_behavior']}")
-        
-        # Clear history for this test
-        history_service.clear_history(session_id)
-        
-        # Simulate conversation
-        print("\n💬 Simulating conversation...")
-        for j, (role, content) in enumerate(scenario["conversation"]):
-            print(f"   {role.upper()}: {content}")
-            
-            if role == "user":
-                # Add user message to history
-                history_service.add_message(session_id, role, content)
-                
-                # Get conversation history
-                conversation_history = history_service.get_conversation_history(session_id)
-                print(f"   📚 History length: {len(conversation_history)} messages")
-                
-                # Test tool decision with enhanced context
-                decision_prompt = prompt_builder.build_tool_decision_prompt(content, conversation_history)
-                decision_json = await llm_client.decide_tool(decision_prompt, conversation_history)
-                
-                tool = decision_json.get("tool")
-                resource = decision_json.get("resource")
-                params = decision_json.get("params", {})
-                
-                print(f"   🔍 Tool decision: tool={tool}, resource={resource}")
-                print(f"   📊 Parameters: {params}")
-                
-                # Add assistant response to history
-                history_service.add_message(session_id, "assistant", content)
-        
-        print(f"   ✅ Test {i+1} completed")
-    
-    print("\n✅ All context-aware selection tests completed!")
-
-async def test_parameter_extraction_with_context():
-    """Test parameter extraction with conversation context."""
-    
-    print("\n🔧 Testing Parameter Extraction with Context")
-    print("=" * 50)
-    
-    # Initialize services
-    llm_client = LLMClient()
-    history_service = ConversationHistoryService()
-    session_id = "param_test"
-    
-    # Test parameter extraction scenarios
-    test_cases = [
-        {
-            "name": "AI Challenge Follow-up",
-            "conversation": [
-                ("user", "Show me active challenges"),
-                ("assistant", "I found several active challenges: AI Challenge #123, Web Development #456."),
-                ("user", "What about AI challenges specifically?")
-            ],
-            "tool": "query-tc-challenges",
-            "expected_params": ["track", "status"]
-        },
-        {
-            "name": "Programming Skills Follow-up",
-            "conversation": [
-                ("user", "What programming skills are available?"),
-                ("assistant", "I found programming skills: Python, JavaScript, Java."),
-                ("user", "Show me more programming skills")
-            ],
-            "tool": "query-tc-skills",
-            "expected_params": ["name"]
-        }
-    ]
-    
-    for i, test_case in enumerate(test_cases):
-        print(f"\n📝 Test {i+1}: {test_case['name']}")
-        
-        # Clear history for this test
-        history_service.clear_history(session_id)
-        
-        # Simulate conversation
-        for role, content in test_case["conversation"]:
-            history_service.add_message(session_id, role, content)
-        
-        # Get conversation history
-        conversation_history = history_service.get_conversation_history(session_id)
-        
-        # Test parameter extraction with context
-        from config.prompt_templates import PromptTemplates
-        
-        # Create context summary
-        context_messages = conversation_history[-4:]
-        context_text = "\n".join([
-            f"{msg['role'].upper()}: {msg['content'][:100]}{'...' if len(msg['content']) > 100 else ''}"
-            for msg in context_messages
-        ])
-        
-        param_extraction_prompt = PromptTemplates.get_tool_param_extraction_prompt(
-            tool_name=test_case["tool"],
-            user_message=test_case["conversation"][-1][1],  # Last user message
-            tool_parameters="Available parameters for the tool",
-            conversation_context=context_text
-        )
-        
-        param_response = await llm_client.complete_json(param_extraction_prompt, conversation_history)
-        params = param_response.get("params", {})
-        
-        print(f"   🔍 Extracted parameters: {params}")
-        print(f"   📊 Expected parameters: {test_case['expected_params']}")
-        
-        # Check if expected parameters are present
-        extracted_param_names = list(params.keys())
-        expected_params = test_case["expected_params"]
-        
-        matches = [param for param in expected_params if param in extracted_param_names]
-        if matches:
-            print(f"   ✅ Found expected parameters: {matches}")
-        else:
-            print(f"   ⚠️ Expected parameters not found: {expected_params}")
-    
-    print("\n✅ All parameter extraction tests completed!")
-
-async def main():
-    """Run all context-aware tests."""
-    try:
-        await test_context_aware_selection()
-        await test_parameter_extraction_with_context()
-        print("\n🎉 All context-aware tests completed successfully!")
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        import traceback
-        traceback.print_exc()
-
-if __name__ == "__main__":
-    asyncio.run(main()) 

scripts/test_full_execution.py

@@ -1,184 +0,0 @@
-#!/usr/bin/env python3
-"""
-Comprehensive test for full tool execution flow.
-This script tests the complete pipeline from user input to tool execution.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-# Add src to path so we can import modules
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-
-# Add root directory to path so we can import config module
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.infrastructure.providers.llm_provider import LLMClient
-from src.application.services.prompt_service import PromptBuilder
-from src.application.services.tool_service import ToolExecutor
-from src.domain.services.conversation_history import ConversationHistoryService
-from src.domain.models.tool import ToolRequest
-from config import settings
-
-async def test_full_execution():
-    """Test the complete execution flow."""
-    
-    print("🧪 Testing Full Execution Flow")
-    print("=" * 50)
-    
-    # Initialize services
-    llm_client = LLMClient()
-    prompt_builder = PromptBuilder()
-    tool_executor = ToolExecutor()
-    history_service = ConversationHistoryService()
-    
-    # Test cases
-    test_cases = [
-        {
-            "name": "Challenge query",
-            "message": "Show me active challenges",
-            "expected_tool": "query-tc-challenges"
-        },
-        {
-            "name": "Skills query", 
-            "message": "What programming skills are available?",
-            "expected_tool": "query-tc-skills"
-        }
-    ]
-    
-    for i, test_case in enumerate(test_cases):
-        print(f"\n📝 Test {i+1}: {test_case['name']}")
-        print(f"   Message: {test_case['message']}")
-        
-        # STEP 1: Tool decision
-        print("   🔍 Step 1: Tool decision...")
-        decision_prompt = prompt_builder.build_tool_decision_prompt(test_case['message'])
-        decision_json = await llm_client.decide_tool(decision_prompt)
-        
-        tool = decision_json.get("tool")
-        resource = decision_json.get("resource")
-        params = decision_json.get("params", {})
-        
-        print(f"   📊 Decision: tool={tool}, resource={resource}, params={params}")
-        
-        # STEP 2: Determine target
-        available_tools = tool_executor.get_available_tools()
-        available_resources = tool_executor.get_available_resources()
-        
-        if tool and tool in available_tools:
-            target_name = tool
-            target_type = "tool"
-        elif resource and resource in available_resources:
-            target_name = resource
-            target_type = "resource"
-        else:
-            print(f"   ❌ Invalid selection: {tool or resource}")
-            continue
-        
-        print(f"   🎯 Target: {target_name} ({target_type})")
-        
-        # STEP 3: Execute tool
-        print("   ⚡ Step 3: Executing tool...")
-        request = ToolRequest(tool=target_name, params=params)
-        
-        try:
-            result = await tool_executor.execute(request)
-            print(f"   📊 Result status: {result.status}")
-            
-            if result.status == "success":
-                print("   ✅ Tool execution successful!")
-                if result.data:
-                    print(f"   📄 Data length: {len(str(result.data))} characters")
-            else:
-                print(f"   ❌ Tool execution failed: {result.message}")
-                
-        except Exception as e:
-            print(f"   ❌ Exception during execution: {e}")
-    
-    print("\n✅ Full execution tests completed!")
-
-async def test_conversation_with_execution():
-    """Test conversation flow with actual tool execution."""
-    
-    print("\n💬 Testing Conversation with Execution")
-    print("=" * 50)
-    
-    # Initialize services
-    llm_client = LLMClient()
-    prompt_builder = PromptBuilder()
-    tool_executor = ToolExecutor()
-    history_service = ConversationHistoryService()
-    session_id = "execution_test"
-    
-    # Simulate a conversation with tool execution
-    conversation = [
-        ("user", "Show me active challenges"),
-        ("user", "What about AI challenges specifically?")
-    ]
-    
-    print("📝 Simulating conversation with execution...")
-    
-    for i, (role, content) in enumerate(conversation):
-        print(f"\n   {role.upper()}: {content}")
-        
-        # Add to history
-        history_service.add_message(session_id, role, content)
-        conversation_history = history_service.get_conversation_history(session_id)
-        
-        # Tool decision with history
-        decision_prompt = prompt_builder.build_tool_decision_prompt(content, conversation_history)
-        decision_json = await llm_client.decide_tool(decision_prompt, conversation_history)
-        
-        tool = decision_json.get("tool")
-        resource = decision_json.get("resource")
-        params = decision_json.get("params", {})
-        
-        print(f"   🔍 Decision: tool={tool}, resource={resource}, params={params}")
-        
-        # Execute if valid
-        if tool or resource:
-            available_tools = tool_executor.get_available_tools()
-            available_resources = tool_executor.get_available_resources()
-            
-            if tool and tool in available_tools:
-                target_name = tool
-            elif resource and resource in available_resources:
-                target_name = resource
-            else:
-                print(f"   ❌ Invalid target: {tool or resource}")
-                continue
-            
-            print(f"   ⚡ Executing: {target_name}")
-            request = ToolRequest(tool=target_name, params=params)
-            
-            try:
-                result = await tool_executor.execute(request)
-                print(f"   📊 Result: {result.status}")
-                
-                if result.status == "success":
-                    print("   ✅ Execution successful!")
-                else:
-                    print(f"   ❌ Execution failed: {result.message}")
-                    
-            except Exception as e:
-                print(f"   ❌ Exception: {e}")
-        
-        # Add assistant response to history
-        history_service.add_message(session_id, "assistant", f"Response to: {content}")
-    
-    print("\n✅ Conversation with execution test completed!")
-
-async def main():
-    """Run all tests."""
-    try:
-        await test_full_execution()
-        await test_conversation_with_execution()
-        print("\n🎉 All execution tests completed successfully!")
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        import traceback
-        traceback.print_exc()
-
-if __name__ == "__main__":
-    asyncio.run(main()) 

scripts/test_tool_selection.py

@@ -1,156 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for tool selection and tool calls with conversation history.
-This script tests the core functionality to ensure everything works correctly.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-# Add src to path so we can import modules
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
-
-# Add root directory to path so we can import config module
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from src.infrastructure.providers.llm_provider import LLMClient
-from src.application.services.prompt_service import PromptBuilder
-from src.domain.services.conversation_history import ConversationHistoryService
-from config import settings
-
-async def test_tool_selection():
-    """Test tool selection with and without conversation history."""
-    
-    print("🧪 Testing Tool Selection")
-    print("=" * 50)
-    
-    # Initialize services
-    llm_client = LLMClient()
-    prompt_builder = PromptBuilder()
-    history_service = ConversationHistoryService()
-    
-    # Test cases
-    test_cases = [
-        {
-            "name": "Simple challenge query",
-            "message": "Show me active challenges",
-            "expected_tool": "query-tc-challenges"
-        },
-        {
-            "name": "Skills query",
-            "message": "What programming skills are available?",
-            "expected_tool": "query-tc-skills"
-        },
-        {
-            "name": "General chat",
-            "message": "Hello, how are you?",
-            "expected_tool": "chat"
-        },
-        {
-            "name": "Member query with handles array",
-            "message": "Show me member abhishek",
-            "expected_tool": "Member_V6_API_Swagger"
-        }
-    ]
-    
-    for i, test_case in enumerate(test_cases):
-        print(f"\n📝 Test {i+1}: {test_case['name']}")
-        print(f"   Message: {test_case['message']}")
-        
-        # Test without history
-        print("   🔍 Testing without conversation history...")
-        decision_prompt = prompt_builder.build_tool_decision_prompt(test_case['message'])
-        decision_json = await llm_client.decide_tool(decision_prompt)
-        
-        tool = decision_json.get("tool")
-        resource = decision_json.get("resource")
-        
-        print(f"   📊 Result: tool={tool}, resource={resource}")
-        
-        # Test with history
-        print("   🔍 Testing with conversation history...")
-        session_id = "test_session"
-        
-        # Add some context to history
-        history_service.add_message(session_id, "user", "I'm looking for challenges")
-        history_service.add_message(session_id, "assistant", "I can help you find challenges. What type are you interested in?")
-        
-        conversation_history = history_service.get_conversation_history(session_id)
-        decision_prompt_with_history = prompt_builder.build_tool_decision_prompt(test_case['message'], conversation_history)
-        decision_json_with_history = await llm_client.decide_tool(decision_prompt_with_history, conversation_history)
-        
-        tool_with_history = decision_json_with_history.get("tool")
-        resource_with_history = decision_json_with_history.get("resource")
-        
-        print(f"   📊 Result with history: tool={tool_with_history}, resource={resource_with_history}")
-        
-        # Check if results are reasonable
-        if tool or resource:
-            print("   ✅ Tool selection working")
-        else:
-            print("   ❌ No tool selected")
-    
-    print("\n✅ Tool selection tests completed!")
-
-async def test_conversation_flow():
-    """Test a complete conversation flow."""
-    
-    print("\n💬 Testing Conversation Flow")
-    print("=" * 50)
-    
-    # Initialize services
-    llm_client = LLMClient()
-    history_service = ConversationHistoryService()
-    session_id = "conversation_test"
-    
-    # Simulate a conversation
-    conversation = [
-        ("user", "Show me active challenges"),
-        ("assistant", "I found several active challenges. Here are some highlights: AI Challenge #123, Web Development #456."),
-        ("user", "What about AI challenges specifically?"),
-        ("assistant", "Looking at the active challenges, here are the AI-specific ones: AI Challenge #123 (Machine Learning), AI Challenge #124 (Computer Vision)."),
-        ("user", "Tell me more about the Machine Learning challenge")
-    ]
-    
-    print("📝 Simulating conversation...")
-    
-    for i, (role, content) in enumerate(conversation):
-        print(f"   {role.upper()}: {content}")
-        
-        if role == "user":
-            # Add user message to history
-            history_service.add_message(session_id, role, content)
-            
-            # Get conversation history
-            conversation_history = history_service.get_conversation_history(session_id)
-            print(f"   📚 History length: {len(conversation_history)} messages")
-            
-            # Test tool decision with history
-            from src.application.services.prompt_service import PromptBuilder
-            prompt_builder = PromptBuilder()
-            decision_prompt = prompt_builder.build_tool_decision_prompt(content, conversation_history)
-            decision_json = await llm_client.decide_tool(decision_prompt, conversation_history)
-            
-            tool = decision_json.get("tool")
-            resource = decision_json.get("resource")
-            print(f"   🔍 Tool decision: tool={tool}, resource={resource}")
-            
-            # Add assistant response to history
-            history_service.add_message(session_id, "assistant", content)
-    
-    print("\n✅ Conversation flow test completed!")
-
-async def main():
-    """Run all tests."""
-    try:
-        await test_tool_selection()
-        await test_conversation_flow()
-        print("\n🎉 All tests completed successfully!")
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        import traceback
-        traceback.print_exc()
-
-if __name__ == "__main__":
-    asyncio.run(main()) 

src/application/services/specs_service.py

@@ -1,486 +0,0 @@
-import json
-import asyncio
-from typing import Dict, List, Any, Optional
-from .catalog_utils import CatalogUtils
-from .stream_client import load_server_config, StreamableHttpMCPClient
-from config.settings import EXCLUDE_AUTH_REQUIRED_ENDPOINTS, EXCLUDE_401_RESPONSES, STRICT_AUTH_FILTERING
-
-
-class SpecsGenerator:
-    """Generates compact API specs from cataloged data and tests functionality."""
-    
-    def __init__(self, catalog_dir: str = "data/catalog"):
-        self.catalog_utils = CatalogUtils(catalog_dir)
-        self.compact_specs = {}
-    
-    async def test_tool_functionality(self, tool_name: str) -> bool:
-        """Test if a tool is working by making a minimal call."""
-        try:
-            server_config = load_server_config()
-            async with StreamableHttpMCPClient(server_config) as client:
-                await client.initialize()
-                
-                # Get tool schema to find a valid test parameter
-                schema = self.catalog_utils.get_tool_schema(tool_name)
-                if not schema:
-                    return False
-                
-                input_schema = schema.get("inputSchema", {})
-                properties = input_schema.get("properties", {})
-                
-                # Find a simple parameter to test with
-                test_params = {}
-                for param_name, param_info in properties.items():
-                    if param_info.get("type") == "string" and not param_info.get("required", False):
-                        test_params[param_name] = "test"
-                        break
-                    elif param_info.get("type") == "number" and not param_info.get("required", False):
-                        test_params[param_name] = 1
-                        break
-                
-                # Make a test call
-                request = self.catalog_utils.format_tool_request(tool_name, test_params)
-                result = await client.call("tools/call", request["params"])
-                
-                # Check if we got a valid response (not an error)
-                return "error" not in result
-                
-        except Exception as e:
-            print(f"❌ Tool {tool_name} test failed: {e}")
-            return False
-    
-    async def test_resource_functionality(self, resource_name: str) -> bool:
-        """Test if a resource (API) is working by checking its endpoints."""
-        try:
-            analysis = self.catalog_utils.get_resource_analysis(resource_name)
-            if not analysis:
-                return False
-            
-            # Check if the analysis has errors
-            if analysis.get("error"):
-                return False
-            
-            # Check if we have endpoints
-            endpoints = analysis.get("endpoints", [])
-            if not endpoints:
-                return False
-            
-            # Check if the API has meaningful content
-            title = analysis.get("title", "")
-            description = analysis.get("description", "")
-            
-            # If it's a basic resource without API content, consider it working
-            if not title and not description and len(endpoints) == 0:
-                return True
-            
-            # For API resources, check if we have valid endpoints
-            return len(endpoints) > 0
-            
-        except Exception as e:
-            print(f"❌ Resource {resource_name} test failed: {e}")
-            return False
-    
-    def generate_compact_tool_spec(self, tool_name: str) -> Optional[Dict[str, Any]]:
-        """Generate compact spec for a tool."""
-        schema = self.catalog_utils.get_tool_schema(tool_name)
-        if not schema:
-            return None
-        
-        description = schema.get("description", "")
-        input_schema = schema.get("inputSchema", {})
-        properties = input_schema.get("properties", {})
-        required = input_schema.get("required", [])
-        
-        # Extract ALL parameters with comprehensive details
-        parameters = {}
-        for param_name, param_info in properties.items():
-            param_type = param_info.get("type", "string")
-            param_desc = param_info.get("description", "")
-            param_enum = param_info.get("enum", [])
-            param_required = param_name in required
-            param_items = param_info.get("items", {})
-            param_format = param_info.get("format", "")
-            param_default = param_info.get("default")
-            
-            # Build comprehensive parameter description
-            desc_parts = []
-            
-            # Add base description
-            if param_desc:
-                base_desc = param_desc.split('.')[0] if '.' in param_desc else param_desc
-                desc_parts.append(base_desc)
-            
-            # Add type information
-            if param_type == "array" and param_items:
-                item_type = param_items.get("type", "string")
-                desc_parts.append(f"Array of {item_type}")
-            else:
-                desc_parts.append(f"Type: {param_type}")
-            
-            # Add format if specified
-            if param_format:
-                desc_parts.append(f"Format: {param_format}")
-            
-            # Add required/optional status
-            desc_parts.append("Required" if param_required else "Optional")
-            
-            # Add enum values if available
-            if param_enum:
-                enum_str = ", ".join([f'"{val}"' for val in param_enum[:8]])  # Show up to 8 values
-                if len(param_enum) > 8:
-                    enum_str += f"... (and {len(param_enum) - 8} more)"
-                desc_parts.append(f"Valid values: {enum_str}")
-            
-            # Add default value if specified
-            if param_default is not None:
-                desc_parts.append(f"Default: {param_default}")
-            
-            # Combine all parts
-            desc = ". ".join(desc_parts)
-            
-            # Clean up description for Python string - escape all special characters
-            desc = (desc.replace('\\', '\\\\')  # Escape backslashes first
-                       .replace('"', '\\"')    # Escape quotes
-                       .replace('\n', ' ')     # Replace newlines
-                       .replace('\r', ' ')     # Replace carriage returns
-                       .replace('`', "'"))     # Replace backticks with single quotes
-            parameters[param_name] = desc
-        
-        # Clean up description for Python string
-        clean_description = description[:200] + "..." if len(description) > 200 else description
-        clean_description = clean_description.replace('"', '\\"').replace('\n', ' ').replace('\r', ' ')
-        
-        return {
-            "description": clean_description,
-            "parameters": parameters
-        }
-    
-    def _extract_endpoint_parameters(self, endpoint: Optional[Dict[str, Any]]) -> Dict[str, str]:
-        """Extract comprehensive parameter information from an OpenAPI endpoint."""
-        if not endpoint:
-            return {}
-        
-        parameters_info = {}
-        endpoint_params = endpoint.get("parameters", [])
-        
-        for param in endpoint_params:
-            param_name = param.get("name", "")
-            param_in = param.get("in", "")
-            param_type = param.get("type", "string")
-            param_desc = param.get("description", "")
-            param_required = param.get("required", False)
-            param_enum = param.get("enum", [])
-            param_format = param.get("format", "")
-            param_default = param.get("default")
-            
-            # Skip path parameters for base URL approach
-            if param_in == "path":
-                continue
-            
-            # Skip body parameters as they're not query params
-            if param_in == "body":
-                continue
-            
-            # Build comprehensive parameter description
-            desc_parts = []
-            
-            # Add base description
-            if param_desc:
-                # Clean up description - remove excessive details but keep key info
-                clean_desc = param_desc.replace('\n', ' ').replace('\r', ' ')
-                if len(clean_desc) > 200:
-                    # Take first 200 chars but try to end at sentence
-                    truncated = clean_desc[:200]
-                    last_period = truncated.rfind('.')
-                    if last_period > 100:  # If there's a period in reasonable range
-                        clean_desc = truncated[:last_period + 1]
-                    else:
-                        clean_desc = truncated + "..."
-                desc_parts.append(clean_desc)
-            
-            # Add type and location information
-            desc_parts.append(f"Type: {param_type}")
-            if param_in:
-                desc_parts.append(f"Location: {param_in}")
-            
-            # Add required/optional status
-            desc_parts.append("Required" if param_required else "Optional")
-            
-            # Add format if specified
-            if param_format:
-                desc_parts.append(f"Format: {param_format}")
-            
-            # Add enum values if available
-            if param_enum:
-                enum_str = ", ".join([f'"{val}"' for val in param_enum[:6]])  # Show up to 6 values
-                if len(param_enum) > 6:
-                    enum_str += f"... (and {len(param_enum) - 6} more)"
-                desc_parts.append(f"Valid values: {enum_str}")
-            
-            # Add default value if specified
-            if param_default is not None:
-                desc_parts.append(f"Default: {param_default}")
-            
-            # Combine all parts
-            desc = ". ".join(desc_parts)
-            
-            # Clean up description for Python string - escape all special characters
-            desc = (desc.replace('\\', '\\\\')  # Escape backslashes first
-                       .replace('"', '\\"')    # Escape quotes
-                       .replace('\n', ' ')     # Replace newlines
-                       .replace('\r', ' ')     # Replace carriage returns
-                       .replace('`', "'"))     # Replace backticks with single quotes
-            
-            if param_name:
-                parameters_info[param_name] = desc
-        
-        return parameters_info
-    
-    def generate_compact_resource_spec(self, resource_name: str) -> Optional[Dict[str, Any]]:
-        """Generate compact spec for a resource."""
-        analysis = self.catalog_utils.get_resource_analysis(resource_name)
-        if not analysis:
-            return None
-        
-        title = analysis.get("title", "")
-        description = analysis.get("description", "")
-        endpoints = analysis.get("endpoints", [])
-        base_url = analysis.get("base_url", "")
-        
-        # Filter out authentication-required endpoints if configured
-        if EXCLUDE_AUTH_REQUIRED_ENDPOINTS:
-            original_count = len(endpoints)
-            filtered_endpoints = []
-            
-            for e in endpoints:
-                requires_auth = e.get("requires_auth", False)
-                
-                # Apply different filtering strategies based on configuration
-                should_exclude = False
-                
-                if EXCLUDE_401_RESPONSES and requires_auth:
-                    should_exclude = True
-                elif STRICT_AUTH_FILTERING:
-                    # More strict filtering - exclude if any auth indicators found
-                    description = e.get("description", "").lower()
-                    summary = e.get("summary", "").lower()
-                    auth_keywords = ["authentication", "authorization", "bearer", "token", "auth", "login", "credential", "secured", "private", "admin"]
-                    should_exclude = any(keyword in description or keyword in summary for keyword in auth_keywords)
-                
-                if not should_exclude:
-                    filtered_endpoints.append(e)
-            
-            filtered_count = len(filtered_endpoints)
-            excluded_count = original_count - filtered_count
-            print(f"    📊 Filtered {excluded_count} auth-required endpoints from {resource_name} ({original_count} → {filtered_count})")
-        else:
-            filtered_endpoints = endpoints
-
-        # Extract key endpoints based on user requirements (base URL approach)
-        key_endpoints = []
-        
-        # For Member API - use base /members endpoint with comprehensive parameters
-        if "member" in resource_name.lower():
-            # Find the main /members endpoint - check both filtered and unfiltered endpoints
-            # since /members is known to work without auth despite having 401 responses
-            members_endpoint = None
-            
-            # First check filtered endpoints
-            for endpoint in filtered_endpoints:
-                if endpoint.get("path") == "/members" and endpoint.get("method") == "GET":
-                    members_endpoint = endpoint
-                    break
-            
-            # If not found in filtered, check original endpoints (known working case)
-            if not members_endpoint:
-                for endpoint in endpoints:
-                    if endpoint.get("path") == "/members" and endpoint.get("method") == "GET":
-                        # Special case: /members works without auth despite 401 responses
-                        if "authentication credential is optional" in endpoint.get("description", "").lower():
-                            members_endpoint = endpoint
-                            break
-            
-            # Extract comprehensive parameter information
-            params_info = self._extract_endpoint_parameters(members_endpoint) if members_endpoint else {}
-            params_desc = f"Available parameters: {', '.join(params_info.keys())}" if params_info else "Get members list with optional filters like handle, page, perPage"
-            
-            key_endpoints.append({
-                "path": "/members",
-                "method": "GET",
-                "full_url": f"{base_url}/members",
-                "description": params_desc,
-                "parameters": params_info
-            })
-        
-        # For Challenges API - use base /challenges endpoint with comprehensive parameters
-        elif "challenge" in resource_name.lower():
-            # Find the main /challenges endpoint to extract real parameters
-            challenges_endpoint = None
-            for endpoint in filtered_endpoints:
-                if endpoint.get("path") == "/challenges" and endpoint.get("method") == "GET":
-                    challenges_endpoint = endpoint
-                    break
-            
-            # Extract comprehensive parameter information
-            params_info = self._extract_endpoint_parameters(challenges_endpoint) if challenges_endpoint else {}
-            params_desc = f"Available parameters: {', '.join(params_info.keys())}" if params_info else "Get challenges list with optional filters like status, track, page, perPage"
-            
-            key_endpoints.append({
-                "path": "/challenges",
-                "method": "GET", 
-                "full_url": f"{base_url}/challenges",
-                "description": params_desc,
-                "parameters": params_info
-            })
-            
-        # For other APIs - extract important base endpoints
-        else:
-            seen_base_paths = set()
-            for endpoint in filtered_endpoints[:5]:  # Limit to first 5
-                path = endpoint.get("path", "")
-                method = endpoint.get("method", "GET")
-                summary = endpoint.get("summary", "")
-                full_url = endpoint.get("full_url", "")
-                
-                # Extract base path (remove path parameters)
-                base_path = path.split('/')[1] if '/' in path and len(path.split('/')) > 1 else path
-                base_path = f"/{base_path}" if not base_path.startswith('/') else base_path
-                
-                if base_path not in seen_base_paths and path and method:
-                    seen_base_paths.add(base_path)
-                    key_endpoints.append({
-                        "path": path,
-                        "method": method,
-                        "summary": summary[:100] + "..." if len(summary) > 100 else summary,
-                        "full_url": full_url
-                    })
-        
-        # Generate compact description
-        if title and description:
-            compact_desc = f"{title}: {description[:150]}..."
-        elif title:
-            compact_desc = title
-        elif description:
-            compact_desc = description[:200] + "..."
-        else:
-            compact_desc = f"API with {len(filtered_endpoints)} endpoints"
-        
-        # Clean up description for Python string
-        compact_desc = compact_desc.replace('"', '\\"').replace('\n', ' ').replace('\r', ' ')
-        
-        return {
-            "description": compact_desc,
-            "key_endpoints": key_endpoints,
-            "total_endpoints": len(filtered_endpoints),
-            "base_url": base_url
-        }
-    
-    async def generate_compact_specs(self) -> Dict[str, Any]:
-        """Generate compact specs for all working tools and resources."""
-        
-        # Test and generate tool specs
-        tools = self.catalog_utils.list_available_tools()
-        working_tools = {}
-        
-        for tool_name in tools:
-            print(f"  🧪 Testing tool: {tool_name}")
-            if await self.test_tool_functionality(tool_name):
-                spec = self.generate_compact_tool_spec(tool_name)
-                if spec:
-                    working_tools[tool_name] = spec
-                    print(f"    ✅ {tool_name} - Working")
-            else:
-                print(f"    ❌ {tool_name} - Not working")
-        
-        # Test and generate resource specs
-        resources = self.catalog_utils.list_available_resources()
-        working_resources = {}
-        
-        for resource_name in resources:
-            print(f"  🧪 Testing resource: {resource_name}")
-            if await self.test_resource_functionality(resource_name):
-                spec = self.generate_compact_resource_spec(resource_name)
-                if spec:
-                    working_resources[resource_name] = spec
-                    print(f"    ✅ {resource_name} - Working")
-            else:
-                print(f"    ❌ {resource_name} - Not working")
-        
-        return {
-            "tools": working_tools,
-            "resources": working_resources
-        }
-    
-    def save_compact_specs(self, specs: Dict[str, Any], output_file: str = "config/compact_api_specs.py"):
-        """Save compact specs to a Python file."""
-        with open(output_file, 'w', encoding='utf-8') as f:
-            f.write("# Generated compact API specs from MCP catalog\n")
-            f.write("# Only includes working tools and resources\n\n")
-            
-            f.write("COMPACT_API_SPECS = {\n")
-            
-            # Write tools
-            f.write("    # Working Tools\n")
-            for tool_name, spec in specs.get("tools", {}).items():
-                f.write(f'    "{tool_name}": {{\n')
-                f.write(f'        "description": "{spec["description"]}",\n')
-                f.write("        \"parameters\": {\n")
-                for param_name, param_desc in spec["parameters"].items():
-                    f.write(f'            "{param_name}": "{param_desc}",\n')
-                f.write("        }\n")
-                f.write("    },\n")
-            
-            # Write resources
-            f.write("\n    # Working Resources\n")
-            for resource_name, spec in specs.get("resources", {}).items():
-                f.write(f'    "{resource_name}": {{\n')
-                f.write(f'        "description": "{spec["description"]}",\n')
-                f.write(f'        "total_endpoints": {spec["total_endpoints"]},\n')
-                if spec.get("base_url"):
-                    f.write(f'        "base_url": "{spec["base_url"]}",\n')
-                f.write("        \"key_endpoints\": [\n")
-                for endpoint in spec["key_endpoints"][:5]:  # Limit to 5 key endpoints
-                    endpoint_line = f'            {{"path": "{endpoint["path"]}", "method": "{endpoint["method"]}"'
-                    if endpoint.get("full_url"):
-                        endpoint_line += f', "full_url": "{endpoint["full_url"]}"'
-                    if endpoint.get("description"):
-                        # Escape description for Python string
-                        escaped_desc = endpoint["description"].replace('"', '\\"')
-                        endpoint_line += f', "description": "{escaped_desc}"'
-                    if endpoint.get("parameters"):
-                        # Write parameters as a nested dictionary
-                        endpoint_line += ', "parameters": {'
-                        param_entries = []
-                        for param_name, param_desc in endpoint["parameters"].items():
-                            # Escape all special characters for Python string
-                            escaped_param_desc = (param_desc.replace('\\', '\\\\')  # Escape backslashes first
-                                                             .replace('"', '\\"')    # Escape quotes
-                                                             .replace('\n', ' ')     # Replace newlines
-                                                             .replace('\r', ' ')     # Replace carriage returns
-                                                             .replace('`', "'"))     # Replace backticks with single quotes
-                            param_entries.append(f'"{param_name}": "{escaped_param_desc}"')
-                        endpoint_line += ', '.join(param_entries)
-                        endpoint_line += '}'
-                    endpoint_line += "},\n"
-                    f.write(endpoint_line)
-                f.write("        ]\n")
-                f.write("    },\n")
-            
-            f.write("}\n")
-        
-        print(f"✅ Compact specs saved to {output_file}")
-    
-    def print_summary(self, specs: Dict[str, Any]):
-        """Print a summary of the generated specs."""
-        tools = specs.get("tools", {})
-        resources = specs.get("resources", {})
-        
-        print(f"\n📊 Compact Specs Summary:")
-        print(f"  🔧 Working Tools: {len(tools)}")
-        for tool_name in tools:
-            print(f"    - {tool_name}")
-        
-        print(f"  📚 Working Resources: {len(resources)}")
-        for resource_name in resources:
-            total_endpoints = resources[resource_name].get("total_endpoints", 0)
-            print(f"    - {resource_name} ({total_endpoints} endpoints)") 

src/infrastructure/logging/logger.py

@@ -1,164 +0,0 @@
-"""
-Structured logging configuration with support for different output formats.
-Provides context-aware logging for better observability.
-"""
-
-import logging
-import logging.handlers
-import json
-import sys
-from typing import Any, Dict, Optional
-from pathlib import Path
-from datetime import datetime
-
-from config.settings import LoggingSettings
-
-
-class StructuredFormatter(logging.Formatter):
-    """JSON formatter for structured logging."""
-    
-    def format(self, record: logging.LogRecord) -> str:
-        """Format log record as JSON."""
-        log_data = {
-            "timestamp": datetime.utcnow().isoformat(),
-            "level": record.levelname,
-            "logger": record.name,
-            "message": record.getMessage(),
-            "module": record.module,
-            "function": record.funcName,
-            "line": record.lineno,
-        }
-        
-        # Add exception info if present
-        if record.exc_info:
-            log_data["exception"] = self.formatException(record.exc_info)
-        
-        # Add extra fields
-        if hasattr(record, "extra"):
-            log_data.update(record.extra)
-        
-        return json.dumps(log_data, default=str)
-
-
-class ContextFilter(logging.Filter):
-    """Add contextual information to log records."""
-    
-    def __init__(self, context: Optional[Dict[str, Any]] = None):
-        super().__init__()
-        self.context = context or {}
-    
-    def filter(self, record: logging.LogRecord) -> bool:
-        """Add context to the log record."""
-        for key, value in self.context.items():
-            setattr(record, key, value)
-        return True
-
-
-class ContextLogger:
-    """Logger wrapper with context support."""
-    
-    def __init__(self, logger: logging.Logger):
-        self.logger = logger
-        self.context: Dict[str, Any] = {}
-    
-    def set_context(self, **kwargs) -> None:
-        """Set logging context."""
-        self.context.update(kwargs)
-    
-    def clear_context(self) -> None:
-        """Clear logging context."""
-        self.context.clear()
-    
-    def _log_with_context(self, level: int, msg: str, *args, **kwargs) -> None:
-        """Log with context."""
-        extra = kwargs.pop("extra", {})
-        extra.update(self.context)
-        kwargs["extra"] = extra
-        self.logger.log(level, msg, *args, **kwargs)
-    
-    def debug(self, msg: str, *args, **kwargs) -> None:
-        """Log debug message with context."""
-        self._log_with_context(logging.DEBUG, msg, *args, **kwargs)
-    
-    def info(self, msg: str, *args, **kwargs) -> None:
-        """Log info message with context."""
-        self._log_with_context(logging.INFO, msg, *args, **kwargs)
-    
-    def warning(self, msg: str, *args, **kwargs) -> None:
-        """Log warning message with context."""
-        self._log_with_context(logging.WARNING, msg, *args, **kwargs)
-    
-    def error(self, msg: str, *args, **kwargs) -> None:
-        """Log error message with context."""
-        self._log_with_context(logging.ERROR, msg, *args, **kwargs)
-    
-    def critical(self, msg: str, *args, **kwargs) -> None:
-        """Log critical message with context."""
-        self._log_with_context(logging.CRITICAL, msg, *args, **kwargs)
-    
-    def exception(self, msg: str, *args, **kwargs) -> None:
-        """Log exception with context."""
-        kwargs["exc_info"] = True
-        self.error(msg, *args, **kwargs)
-
-
-def setup_logger(
-    name: str = "topcoder_mcp_agent",
-    config: Optional[LoggingSettings] = None
-) -> ContextLogger:
-    """
-    Set up application logger with the given configuration.
-    
-    Args:
-        name: Logger name
-        config: Logging configuration
-        
-    Returns:
-        Configured context logger
-    """
-    if config is None:
-        config = LoggingSettings()
-    
-    # Create logger
-    logger = logging.getLogger(name)
-    logger.setLevel(getattr(logging, config.level.upper()))
-    
-    # Clear existing handlers
-    logger.handlers.clear()
-    
-    # Create formatter
-    if config.file_path:
-        # Use structured JSON formatter for file output
-        formatter = StructuredFormatter()
-    else:
-        # Use simple formatter for console output
-        formatter = logging.Formatter(config.format)
-    
-    # Create handlers
-    if config.file_path:
-        # File handler with rotation
-        handler = logging.handlers.RotatingFileHandler(
-            filename=config.file_path,
-            maxBytes=config.max_file_size,
-            backupCount=config.backup_count,
-            encoding="utf-8"
-        )
-    else:
-        # Console handler
-        handler = logging.StreamHandler(sys.stdout)
-    
-    handler.setFormatter(formatter)
-    handler.setLevel(getattr(logging, config.level.upper()))
-    
-    # Add handler to logger
-    logger.addHandler(handler)
-    
-    # Prevent propagation to root logger
-    logger.propagate = False
-    
-    return ContextLogger(logger)
-
-
-def get_logger(name: str) -> ContextLogger:
-    """Get a logger instance with the given name."""
-    return ContextLogger(logging.getLogger(name))

src/infrastructure/providers/long_field_manager.py

@@ -0,0 +1,247 @@
+import json
+import os
+from typing import Any, Dict, List, Tuple
+
+from config import settings
+
+
+class LongFieldManager:
+    """
+    Detects long fields in tool/resource responses and manages a human/LLM-editable
+    configuration file that records which fields are considered long per tool/resource.
+
+    Behavior:
+    - By default, long fields are omitted from the data passed to the final LLM summarization.
+    - If the user's query explicitly asks for the long field(s), include them.
+    """
+
+    def __init__(self, config_path: str = "config/long_fields_config.json"):
+        self.config_path = config_path
+        self._config = self._load_config()
+
+        # Thresholds from settings.py (human editable via env)
+        self.max_string_chars = getattr(settings, "LONG_FIELD_STRING_MAX_CHARS", 800)
+        self.max_array_items = getattr(settings, "LONG_FIELD_ARRAY_MAX_ITEMS", 50)
+        self.max_array_bytes = getattr(settings, "LONG_FIELD_ARRAY_MAX_BYTES", 50_000)
+
+        # Simple keywords that indicate explicit request for full/long content
+        self.request_keywords = [
+            "full", "complete", "entire", "expand", "show more", "all items",
+            "full description", "whole", "everything", "detailed", "complete list"
+        ]
+
+    # -------- Config management --------
+    def _load_config(self) -> Dict[str, Dict[str, List[str]]]:
+        if not os.path.exists(self.config_path):
+            return {"tools": {}, "resources": {}}
+        try:
+            with open(self.config_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                # Ensure shape
+                if not isinstance(data, dict):
+                    return {"tools": {}, "resources": {}}
+                data.setdefault("tools", {})
+                data.setdefault("resources", {})
+                return data
+        except Exception:
+            return {"tools": {}, "resources": {}}
+
+    def _save_config(self) -> None:
+        os.makedirs(os.path.dirname(self.config_path), exist_ok=True)
+        with open(self.config_path, "w", encoding="utf-8") as f:
+            json.dump(self._config, f, indent=2, ensure_ascii=False)
+
+    def record_long_field(self, kind: str, name: str, field_path: str) -> None:
+        """Record a long field path under a tool/resource in config."""
+        if kind not in ("tools", "resources"):
+            return
+        
+        # Skip recording paths that would break response structure:
+        # 1. Top-level single segments (likely containers)
+        # 2. Paths that are common response containers used by filter_tool_response
+        if self._is_structural_path(field_path):
+            return
+            
+        bucket = self._config.setdefault(kind, {})
+        fields = bucket.setdefault(name, [])
+        if field_path not in fields:
+            fields.append(field_path)
+            self._save_config()
+
+    def _is_structural_path(self, field_path: str) -> bool:
+        """
+        Determine if a field path is structural (container) and should not be redacted.
+        Structural paths are those that contain the main response data.
+        """
+        # Only consider actual container names as structural
+        # These are the top-level response containers, not field names
+        if field_path in ("data", "items", "records", "results", "members", "challenges"):
+            return True
+            
+        return False
+
+    def get_known_long_fields(self, kind: str, name: str) -> List[str]:
+        return self._config.get(kind, {}).get(name, [])
+
+    # -------- Detection --------
+    def _is_long_string(self, value: Any) -> bool:
+        return isinstance(value, str) and len(value) > self.max_string_chars
+
+    def _serialized_size(self, value: Any) -> int:
+        try:
+            return len(json.dumps(value, ensure_ascii=False))
+        except Exception:
+            return 0
+
+    def _is_long_array(self, value: Any) -> bool:
+        if not isinstance(value, list):
+            return False
+        # Check multiple criteria for "long" arrays
+        if len(value) > self.max_array_items:
+            return True
+        if self._serialized_size(value) > self.max_array_bytes:
+            return True
+        # For string arrays, also check total character count
+        if value and all(isinstance(x, str) for x in value):
+            total_chars = sum(len(x) for x in value)
+            if total_chars > self.max_string_chars:
+                return True
+        return False
+
+    def _collect_long_fields(self, data: Any, parent_path: str = "") -> List[Tuple[str, str]]:
+        """
+        Traverse data and collect (path, kind) where kind in {"string","array"} is long.
+        Paths are expressed as dot paths; list indices are not included to keep paths stable.
+        """
+        found: List[Tuple[str, str]] = []
+
+        if isinstance(data, dict):
+            for k, v in data.items():
+                path = f"{parent_path}.{k}" if parent_path else k
+                if self._is_long_string(v):
+                    found.append((path, "string"))
+                elif self._is_long_array(v):
+                    found.append((path, "array"))
+                else:
+                    # Recurse into nested structures
+                    found.extend(self._collect_long_fields(v, path))
+        elif isinstance(data, list):
+            # Check if the list itself is long
+            if self._is_long_array(data):
+                found.append((parent_path, "array"))
+            else:
+                # For arrays, examine first few items to find nested long fields
+                for i, item in enumerate(data[:3]):  # Sample first 3 items
+                    if isinstance(item, dict):
+                        # For dict items in arrays, check their fields
+                        for k, v in item.items():
+                            # For array items, use just the field name, not parent_path.field_name
+                            # This matches how the redaction will work
+                            field_path = k
+                            if self._is_long_string(v):
+                                found.append((field_path, "string"))
+                            elif self._is_long_array(v):
+                                found.append((field_path, "array"))
+                            else:
+                                # Recurse deeper
+                                found.extend(self._collect_long_fields(v, field_path))
+
+        return found
+
+    # -------- Query intent for long fields --------
+    def _query_requests_long_fields(self, query: str, field_paths: List[str]) -> bool:
+        q = (query or "").lower()
+        # Keyword signal
+        if any(kw in q for kw in self.request_keywords):
+            return True
+        # Direct field-name signal: match last segment of a path
+        last_segments = {p.split(".")[-1].lower() for p in field_paths}
+        return any(seg in q for seg in last_segments if seg)
+
+    # -------- Public API --------
+    def prepare_data_for_prompt(
+        self,
+        kind: str,  # "tools" or "resources"
+        name: str,
+        data: Any,
+        user_query: str,
+    ) -> Any:
+        """
+        Return data with long fields omitted unless the query explicitly requests them.
+        Also records detected long fields into the editable config file.
+        """
+        if not isinstance(data, (dict, list)):
+            return data
+
+        # Detect long fields present in this payload
+        detected = self._collect_long_fields(data)
+        detected_paths = [p for p, _ in detected]
+
+        # Persist discovered long fields (with structural filtering)
+        recorded_paths = []
+        for p in detected_paths:
+            if not self._is_structural_path(p):
+                self.record_long_field(kind, name, p)
+                recorded_paths.append(p)
+
+        # Combine with known long fields from config
+        known_paths = self.get_known_long_fields(kind, name)
+        all_long_paths = sorted(set(recorded_paths + known_paths))
+
+        if not all_long_paths:
+            return data
+
+        # Decide whether to include long fields based on user query
+        include_long = self._query_requests_long_fields(user_query, all_long_paths)
+
+        if include_long:
+            return data  # keep as-is
+
+        # Otherwise, remove or redact long fields from data
+        return self._redact_long_fields(data, set(all_long_paths))
+
+    def _redact_long_fields(self, data: Any, long_paths: set, parent_path: str = "") -> Any:
+        """
+        Redact only the exact long field paths, not their ancestors.
+        This preserves containers (e.g., keep 'members' list, but redact 'members.skills').
+        """
+        # If the current container itself is the long field, replace it wholesale
+        if parent_path and parent_path in long_paths:
+            return {"_omitted": True, "reason": "long_field", "path": parent_path}
+
+        if isinstance(data, dict):
+            out = {}
+            for k, v in data.items():
+                path = f"{parent_path}.{k}" if parent_path else k
+                if path in long_paths:
+                    out[k] = {"_omitted": True, "reason": "long_field", "path": path}
+                else:
+                    out[k] = self._redact_long_fields(v, long_paths, path)
+            return out
+        elif isinstance(data, list):
+            # Check if this list itself should be redacted
+            if parent_path in long_paths:
+                return {"_omitted": True, "reason": "long_field", "path": parent_path}
+            
+            # Otherwise, process each item in the list
+            result = []
+            for item in data:
+                if isinstance(item, dict):
+                    # For dict items, check each field for redaction
+                    redacted_item = {}
+                    for k, v in item.items():
+                        # For array items, check against the field name directly
+                        if k in long_paths:
+                            redacted_item[k] = {"_omitted": True, "reason": "long_field", "path": k}
+                        else:
+                            redacted_item[k] = self._redact_long_fields(v, long_paths, k)
+                    result.append(redacted_item)
+                else:
+                    result.append(self._redact_long_fields(item, long_paths, parent_path))
+            return result
+        return data
+
+
+# Global singleton
+long_field_manager = LongFieldManager()
+

src/infrastructure/providers/response_filter.py

@@ -417,25 +417,30 @@ def _filter_parsed_data(tool: str, data: dict | list) -> dict | list:
             }
     
     # Handle member-related tools
-    elif "member" in tool.lower():
+    else:
+        # Generic handling for non-specialized tools/resources
         if isinstance(data, dict):
-            members = data.get("data", [])
-            total = data.get("total", 0)
-            
-            return {
-                "summary": {
-                    "total_members": total,
-                    "members_returned": len(members)
-                },
-                "members": members[:5] if isinstance(members, list) else []
-            }
+            # Prefer common list containers
+            for key in ("data", "items", "records", "results"):
+                val = data.get(key)
+                if isinstance(val, list):
+                    lst = val
+                    return {
+                        "summary": {
+                            "total_items": len(lst),
+                            "items_returned": min(5, len(lst))
+                        },
+                        "items": lst[:5]
+                    }
+            # If a single object, just return as-is
+            return data
         elif isinstance(data, list):
             return {
                 "summary": {
-                    "total_members": len(data),
-                    "members_returned": min(5, len(data))
+                    "total_items": len(data),
+                    "items_returned": min(5, len(data))
                 },
-                "members": data[:5]
+                "items": data[:5]
             }
     
     # Default fallback - return data as is but limit lists

src/presentation/gradio_interface.py

@@ -9,6 +9,7 @@ from config.system_prompts import topcoder_system_prompt
 from config.static_responses import REJECT_RESPONSE
 from config.prompt_templates import PromptTemplates
 from src.infrastructure.providers.response_filter import filter_tool_response
+from src.infrastructure.providers.long_field_manager import long_field_manager
 
 llm_client = LLMClient()
 tool_executor = ToolExecutor()
@@ -111,6 +112,22 @@ async def agent_response(user_message, history):
     # STEP 8: Filter response before summarizing
     filtered_data = filter_tool_response(target_name, tool_result.data)
 
+    # Avoid hardcoding member-specific wrapping; rely on generic filter output
+
+    # STEP 8b: Omit long fields by default unless the query explicitly asks for them.
+    # Map target kind for config bucketing
+    kind = "tools" if target_type == "tool" else "resources"
+    filtered_or_redacted = long_field_manager.prepare_data_for_prompt(
+        kind=kind,
+        name=target_name,
+        data=filtered_data,
+        user_query=user_message,
+    )
+
+    # Ensure we never return empty data due to redaction; keep a minimal summary fallback
+    if not filtered_or_redacted or (isinstance(filtered_or_redacted, dict) and len(filtered_or_redacted.keys()) == 0):
+        filtered_or_redacted = {"summary": "Data available with long fields omitted by default.", "context": context}
+
     # Build dynamic summarization prompt based on the target type
     if target_type == "tool":
         context = f"Tool used: {target_name}"
@@ -118,7 +135,7 @@ async def agent_response(user_message, history):
         context = f"Resource used: {target_name}"
     
     summarization_prompt = PromptTemplates.get_gradio_response_summarization_prompt(
-        user_message, context, filtered_data
+        user_message, context, filtered_or_redacted
     )
 
     final_response = await llm_client.chat([