search

    try:
        # Process user input to generate knowledge
        knowledge = knowledge_processor.process_user_input(request.user_input)

        # Store the generated knowledge in vector database
        ids = vector_store.add_texts(
            texts=knowledge["text_chunks"],
            metadata=knowledge["metadata"]
        )

        return {
            "message": "Successfully generated and stored agent knowledge",
            "structured_knowledge": knowledge["structured_knowledge"],
            "vector_ids": ids
        }
    except Exception as e:
        logger.error(f"Knowledge generation error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/search")
async def search(request: QueryRequest):
    try:
        results = vector_store.search(request.query, k=request.k)
        return {"results": results}
    except Exception as e:
        logger.error(f"Search error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/add-texts")
async def add_texts(request: AddTextsRequest):
    try:
        ids = vector_store.add_texts(request.texts, request.metadata)
        return {"ids": ids}
    except Exception as e:
        logger.error(f"Add texts error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")
async def health_check():
    return {"status": "healthy"}

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "main:app",
        host=config['api']['host'],
        port=config['api']['port'],


        return {
            "message": "Successfully generated and stored agent knowledge",
            "structured_knowledge": knowledge["structured_knowledge"],
            "vector_ids": ids
        }
    except Exception as e:
        logger.error(f"Knowledge generation error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/search")
async def search(request: QueryRequest):
    try:
        results = vector_store.search(request.query, k=request.k)
        return {"results": results}
    except Exception as e:
        logger.error(f"Search error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/add-texts")
async def add_texts(request: AddTextsRequest):
    try:
        ids = vector_store.add_texts(request.texts, request.metadata)
        return {"ids": ids}
    except Exception as e:
        logger.error(f"Add texts error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")
async def health_check():
    return {"status": "healthy"}

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "main:app",
        host=config['api']['host'],
        port=config['api']['port'],
        reload=True
    )


@app.post("/search")
async def search(request: QueryRequest):
    try:
        results = vector_store.search(request.query, k=request.k)
        return {"results": results}
    except Exception as e:
        logger.error(f"Search error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/add-texts")
async def add_texts(request: AddTextsRequest):
    try:
        ids = vector_store.add_texts(request.texts, request.metadata)
        return {"ids": ids}
    except Exception as e:
        logger.error(f"Add texts error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")
async def health_check():
    return {"status": "healthy"}

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "main:app",
        host=config['api']['host'],
        port=config['api']['port'],
        reload=True
    )


src.chatgpt.openai

from openai import OpenAI
client = OpenAI()

completion = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "developer", "content": "You are a helpful assistant."},
        {
            "role": "user",
            "content": "Write a haiku about recursion in programming."
        }
    ]
)

print(completion.choices[0].message)


from openai import OpenAI
from typing import Dict, List, Optional
import json
from loguru import logger
import os

class KnowledgeProcessor:
    def __init__(self, api_key: Optional[str] = None):
        """Initialize the knowledge processor with OpenAI client."""
        # Try to get API key from environment if not provided
        self.api_key = api_key or os.getenv('OPENAI_API_KEY')
        if not self.api_key:
            logger.warning("No OpenAI API key provided. Using mock responses for testing.")
        self.client = OpenAI(api_key=self.api_key)

    def process_user_input(self, user_input: str) -> Dict:
        """Process user input to generate agent knowledge and tools."""
        try:
            # Generate structured knowledge from user input
            completion = self.client.chat.completions.create(
                model="gpt-4-1106-preview",
                messages=[
                    {"role": "system", "content": """You are an AI assistant that helps generate structured knowledge and tools for an AI agent.
                    Given user input about what knowledge/capabilities an agent should have, generate:
                    1. Core knowledge areas
                    2. Required tools and functions
                    3. Example queries/tasks the agent should handle

                    Your response must be a valid JSON object with the following structure:
                    {
                        "core_knowledge": [{"title": "string", "description": "string"}],
                        "tools": [{"name": "string", "description": "string"}],
                        "examples": [{"query": "string", "expected_response": "string"}]
                    }"""},
                    {"role": "user", "content": user_input}
                ],
                response_format={"type": "json_object"}
            )

            # Parse the structured knowledge
            knowledge_base = json.loads(completion.choices[0].message.content)

            # Generate embeddings-ready text chunks
            text_chunks = self._generate_text_chunks(knowledge_base)

            return {
                "structured_knowledge": knowledge_base,
                "text_chunks": text_chunks,
                "metadata": self._generate_metadata(knowledge_base)
            }

        except Exception as e:
            logger.error(f"Error processing user input: {e}")
            raise

    def _generate_text_chunks(self, knowledge_base: Dict) -> List[str]:
        """Convert structured knowledge into text chunks for embedding."""
        chunks = []

        # Process core knowledge areas
        for area in knowledge_base.get("core_knowledge", []):
            chunks.append(f"Core Knowledge - {area['title']}: {area['description']}")

        # Process tools and functions
        for tool in knowledge_base.get("tools", []):
            chunks.append(f"Tool - {tool['name']}: {tool['description']}")
            if 'usage_example' in tool:
                chunks.append(f"Tool Usage: {tool['usage_example']}")

        # Process example queries/tasks
        for example in knowledge_base.get("examples", []):
            chunks.append(f"Example Task: {example['query']} -> {example['expected_response']}")

        return chunks

    def _generate_metadata(self, knowledge_base: Dict) -> List[Dict]:
        """Generate metadata for each text chunk."""
        metadata = []

        # Add metadata for each chunk corresponding to _generate_text_chunks
        for area in knowledge_base.get("core_knowledge", []):
            metadata.append({
                "type": "core_knowledge",
                "title": area["title"]
            })

        for tool in knowledge_base.get("tools", []):
            metadata.append({
                "type": "tool",
                "name": tool["name"]
            })
            if 'usage_example' in tool:
                metadata.append({
                    "type": "tool_usage",
                    "name": tool["name"]
                })

        for example in knowledge_base.get("examples", []):
            metadata.append({
                "type": "example",
                "query": example["query"]
            })

        return metadata


from openai import OpenAI
from typing import Dict, List, Optional
import json
from loguru import logger
import os

class KnowledgeProcessor:
    def __init__(self, api_key: Optional[str] = None):
        """Initialize the knowledge processor with OpenAI client."""
        # Try to get API key from environment if not provided
        self.api_key = api_key or os.getenv('OPENAI_API_KEY')
        if not self.api_key:
            logger.warning("No OpenAI API key provided. Using mock responses for testing.")
        self.client = OpenAI(api_key=self.api_key)

    def process_user_input(self, user_input: str) -> Dict:
        """Process user input to generate agent knowledge and tools."""
        try:
            # Generate structured knowledge from user input
            completion = self.client.chat.completions.create(
                model="gpt-4-1106-preview",
                messages=[
                    {"role": "system", "content": """You are an AI assistant that helps generate structured knowledge and tools for an AI agent.
                    Given user input about what knowledge/capabilities an agent should have, generate:
                    1. Core knowledge areas
                    2. Required tools and functions
                    3. Example queries/tasks the agent should handle

                    Your response must be a valid JSON object with the following structure:
                    {
                        "core_knowledge": [{"title": "string", "description": "string"}],
                        "tools": [{"name": "string", "description": "string"}],
                        "examples": [{"query": "string", "expected_response": "string"}]
                    }"""},


from openai import OpenAI
from typing import Dict, List, Optional
import json
from loguru import logger
import os

class KnowledgeProcessor:
    def __init__(self, api_key: Optional[str] = None):
        """Initialize the knowledge processor with OpenAI client."""
        # Try to get API key from environment if not provided
        self.api_key = api_key or os.getenv('OPENAI_API_KEY')
        if not self.api_key:
            logger.warning("No OpenAI API key provided. Using mock responses for testing.")
        self.client = OpenAI(api_key=self.api_key)

    def process_user_input(self, user_input: str) -> Dict:
        """Process user input to generate agent knowledge and tools."""
        try:
            # Generate structured knowledge from user input
            completion = self.client.chat.completions.create(
                model="gpt-4-1106-preview",
                messages=[
                    {"role": "system", "content": """You are an AI assistant that helps generate structured knowledge and tools for an AI agent.
                    Given user input about what knowledge/capabilities an agent should have, generate:
                    1. Core knowledge areas
                    2. Required tools and functions
                    3. Example queries/tasks the agent should handle

                    Your response must be a valid JSON object with the following structure:
                    {
                        "core_knowledge": [{"title": "string", "description": "string"}],
                        "tools": [{"name": "string", "description": "string"}],
                        "examples": [{"query": "string", "expected_response": "string"}]
                    }"""},
                    {"role": "user", "content": user_input}
                ],
                response_format={"type": "json_object"}
            )

            # Parse the structured knowledge
            knowledge_base = json.loads(completion.choices[0].message.content)

            # Generate embeddings-ready text chunks
            text_chunks = self._generate_text_chunks(knowledge_base)

            return {
                "structured_knowledge": knowledge_base,
                "text_chunks": text_chunks,
                "metadata": self._generate_metadata(knowledge_base)
            }

        except Exception as e:
            logger.error(f"Error processing user input: {e}")
            raise

    def _generate_text_chunks(self, knowledge_base: Dict) -> List[str]:
        """Convert structured knowledge into text chunks for embedding."""
        chunks = []

        # Process core knowledge areas
        for area in knowledge_base.get("core_knowledge", []):
            chunks.append(f"Core Knowledge - {area['title']}: {area['description']}")

        # Process tools and functions
        for tool in knowledge_base.get("tools", []):
            chunks.append(f"Tool - {tool['name']}: {tool['description']}")
            if 'usage_example' in tool:
                chunks.append(f"Tool Usage: {tool['usage_example']}")

        # Process example queries/tasks
        for example in knowledge_base.get("examples", []):
            chunks.append(f"Example Task: {example['query']} -> {example['expected_response']}")

        return chunks


                ],
                response_format={"type": "json_object"}
            )

            # Parse the structured knowledge
            knowledge_base = json.loads(completion.choices[0].message.content)

            # Generate embeddings-ready text chunks
            text_chunks = self._generate_text_chunks(knowledge_base)

            return {
                "structured_knowledge": knowledge_base,
                "text_chunks": text_chunks,
                "metadata": self._generate_metadata(knowledge_base)
            }

        except Exception as e:
            logger.error(f"Error processing user input: {e}")
            raise

    def _generate_text_chunks(self, knowledge_base: Dict) -> List[str]:
        """Convert structured knowledge into text chunks for embedding."""
        chunks = []

        # Process core knowledge areas
        for area in knowledge_base.get("core_knowledge", []):
            chunks.append(f"Core Knowledge - {area['title']}: {area['description']}")

        # Process tools and functions
        for tool in knowledge_base.get("tools", []):
            chunks.append(f"Tool - {tool['name']}: {tool['description']}")
            if 'usage_example' in tool:
                chunks.append(f"Tool Usage: {tool['usage_example']}")

        # Process example queries/tasks
        for example in knowledge_base.get("examples", []):
            chunks.append(f"Example Task: {example['query']} -> {example['expected_response']}")

        return chunks

    def _generate_metadata(self, knowledge_base: Dict) -> List[Dict]:
        """Generate metadata for each text chunk."""
        metadata = []

        # Add metadata for each chunk corresponding to _generate_text_chunks
        for area in knowledge_base.get("core_knowledge", []):
            metadata.append({
                "type": "core_knowledge",
                "title": area["title"]
            })

        for tool in knowledge_base.get("tools", []):
            metadata.append({
                "type": "tool",
                "name": tool["name"]
            })
            if 'usage_example' in tool:
                metadata.append({
                    "type": "tool_usage",


    def _generate_text_chunks(self, knowledge_base: Dict) -> List[str]:
        """Convert structured knowledge into text chunks for embedding."""
        chunks = []

        # Process core knowledge areas
        for area in knowledge_base.get("core_knowledge", []):
            chunks.append(f"Core Knowledge - {area['title']}: {area['description']}")

        # Process tools and functions
        for tool in knowledge_base.get("tools", []):
            chunks.append(f"Tool - {tool['name']}: {tool['description']}")
            if 'usage_example' in tool:
                chunks.append(f"Tool Usage: {tool['usage_example']}")

        # Process example queries/tasks
        for example in knowledge_base.get("examples", []):
            chunks.append(f"Example Task: {example['query']} -> {example['expected_response']}")

        return chunks

    def _generate_metadata(self, knowledge_base: Dict) -> List[Dict]:
        """Generate metadata for each text chunk."""
        metadata = []

        # Add metadata for each chunk corresponding to _generate_text_chunks
        for area in knowledge_base.get("core_knowledge", []):
            metadata.append({
                "type": "core_knowledge",
                "title": area["title"]
            })

        for tool in knowledge_base.get("tools", []):
            metadata.append({
                "type": "tool",
                "name": tool["name"]
            })
            if 'usage_example' in tool:
                metadata.append({
                    "type": "tool_usage",
                    "name": tool["name"]
                })

        for example in knowledge_base.get("examples", []):
            metadata.append({
                "type": "example",
                "query": example["query"]
            })

        return metadata


tests/conftest.py

import pytest
import os
import sys
from unittest.mock import Mock, patch
import numpy as np

# Add the src directory to the Python path
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Create mock for sentence transformers before any tests run
@pytest.fixture(autouse=True)
def mock_dependencies():
    """Mock external dependencies for all tests."""
    mock_sentence_transformer = Mock()
    mock_sentence_transformer.encode.return_value = np.random.rand(1, 384).astype('float32')

    with patch.dict('sys.modules', {
        'sentence_transformers': Mock(),
        'sentence_transformers.SentenceTransformer': Mock(return_value=mock_sentence_transformer),
        'tensorflow': Mock(),
        'torch': Mock()


import pytest
import numpy as np
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.database.vector_store import VectorStore

@pytest.fixture
def vector_store():
    """Create a real VectorStore instance."""
    return VectorStore()

def test_preprocess_text(vector_store):
    """Test text preprocessing."""
    text = "Hello, World! This is a TEST."
    words = vector_store._preprocess_text(text)
    assert words == ['hello', 'world', 'this', 'is', 'a', 'test']

def test_basic_embedding_flow(vector_store):
    """Test the basic flow of adding and searching texts."""


import pytest
import json
import os
from datetime import datetime
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.utils.logger import SystemLogger

@pytest.fixture
def temp_log_files(tmp_path):
    actions_log = tmp_path / "actions.log"
    thoughts_log = tmp_path / "thoughts.log"
    return str(actions_log), str(thoughts_log)

@pytest.fixture
def system_logger(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    return SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

def test_log_file_creation(temp_log_files):


import pytest
from fastapi.testclient import TestClient
import os
import sys
from dotenv import load_dotenv

# Load environment variables before importing app
load_dotenv()

# Ensure we have an API key for tests
if not os.getenv('OPENAI_API_KEY'):
    pytest.skip("Skipping tests that require OpenAI API key", allow_module_level=True)

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.api.main import app

@pytest.fixture
def client():
    """Create a test client."""
    return TestClient(app)


import pytest
from unittest.mock import patch, MagicMock
import json
import os
import sys
from dotenv import load_dotenv

# Load environment variables from .env
load_dotenv()

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.chatgpt.knowledge_processor import KnowledgeProcessor

@pytest.fixture
def mock_openai():
    """Create a mock OpenAI client."""
    mock_response = {
        "choices": [{
            "message": {
                "content": json.dumps({
                    "core_knowledge": [


from typing import List, Dict, Any, Optional
import faiss
import numpy as np
from loguru import logger
import yaml
import os
import re
from collections import Counter

class VectorStore:
    def __init__(self, config_path: str = "config.yaml"):
        with open(config_path, 'r') as f:
            self.config = yaml.safe_load(f)

        self.index = None
        self.metadata = {}
        self.dimension = 100  # Fixed dimension for our simple embeddings
        self.vocabulary = {}  # Word to index mapping
        self.next_word_index = 0

    def _preprocess_text(self, text: str) -> List[str]:


from typing import List, Dict, Any, Tuple
import pandas as pd
import json
from loguru import logger

class DataProcessor:
    @staticmethod
    def process_file(file_path: str, file_format: str) -> Tuple[List[str], List[Dict]]:
        """Process input file and return texts and metadata."""
        try:
            if file_format == 'csv':
                return DataProcessor._process_csv(file_path)
            elif file_format == 'json':
                return DataProcessor._process_json(file_path)
            elif file_format == 'txt':
                return DataProcessor._process_txt(file_path)
            else:
                raise ValueError(f"Unsupported file format: {file_format}")
        except Exception as e:
            logger.error(f"Error processing file: {e}")
            raise


import json
from datetime import datetime
from typing import Any, Dict, Optional
import os
from pathlib import Path

class SystemLogger:
    def __init__(self, actions_log_path: str = "actions.log", thoughts_log_path: str = "thoughts.log"):
        """Initialize the system logger."""
        self.actions_log_path = actions_log_path
        self.thoughts_log_path = thoughts_log_path
        self._ensure_log_files_exist()

    def _ensure_log_files_exist(self):
        """Ensure log files exist and create them if they don't."""
        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {


from fastapi import FastAPI, UploadFile, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
from typing import List, Optional, Dict
import yaml
from loguru import logger
import sys
import os
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database.vector_store import VectorStore
from chatgpt.knowledge_processor import KnowledgeProcessor

app = FastAPI(title="RAG Vector Database API")

# CORS middleware
app.add_middleware(


from openai import OpenAI
from typing import Dict, List, Optional
import json
from loguru import logger
import os

class KnowledgeProcessor:
    def __init__(self, api_key: Optional[str] = None):
        """Initialize the knowledge processor with OpenAI client."""
        # Try to get API key from environment if not provided
        self.api_key = api_key or os.getenv('OPENAI_API_KEY')
        if not self.api_key:
            logger.warning("No OpenAI API key provided. Using mock responses for testing.")
        self.client = OpenAI(api_key=self.api_key)

    def process_user_input(self, user_input: str) -> Dict:
        """Process user input to generate agent knowledge and tools."""
        try:
            # Generate structured knowledge from user input
            completion = self.client.chat.completions.create(
                model="gpt-4-1106-preview",