tests.conftest

import pytest
import os
import sys
from unittest.mock import Mock, patch
import numpy as np

# Add the src directory to the Python path
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Create mock for sentence transformers before any tests run
@pytest.fixture(autouse=True)
def mock_dependencies():
    """Mock external dependencies for all tests."""
    mock_sentence_transformer = Mock()
    mock_sentence_transformer.encode.return_value = np.random.rand(1, 384).astype('float32')

    with patch.dict('sys.modules', {
        'sentence_transformers': Mock(),
        'sentence_transformers.SentenceTransformer': Mock(return_value=mock_sentence_transformer),
        'tensorflow': Mock(),
        'torch': Mock()
    }):
        yield

@pytest.fixture
def test_data_dir(tmp_path):
    """Create a temporary directory for test data."""
    data_dir = tmp_path / "test_data"
    data_dir.mkdir()
    return data_dir

@pytest.fixture
def test_config():
    """Return a test configuration dictionary."""
    return {
        'api': {
            'host': 'localhost',
            'port': 8000
        },
        'embedding': {
            'default_model': 'sentence-transformers/all-MiniLM-L6-v2',
            'fallback_model': 'sentence-transformers/paraphrase-MiniLM-L3-v2',
            'dimension': 384
        },
        'vector_store': {
            'index_type': 'flat',
            'max_batch_size': 100
        },
        'openai': {
            'model': 'gpt-4',
            'temperature': 0.7,
            'max_tokens': 2000
        }
    }


import pytest
import os
import sys
from unittest.mock import Mock, patch
import numpy as np

# Add the src directory to the Python path
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Create mock for sentence transformers before any tests run
@pytest.fixture(autouse=True)
def mock_dependencies():
    """Mock external dependencies for all tests."""
    mock_sentence_transformer = Mock()
    mock_sentence_transformer.encode.return_value = np.random.rand(1, 384).astype('float32')

    with patch.dict('sys.modules', {
        'sentence_transformers': Mock(),
        'sentence_transformers.SentenceTransformer': Mock(return_value=mock_sentence_transformer),
        'tensorflow': Mock(),
        'torch': Mock()
    }):
        yield

@pytest.fixture
def test_data_dir(tmp_path):
    """Create a temporary directory for test data."""
    data_dir = tmp_path / "test_data"
    data_dir.mkdir()
    return data_dir

@pytest.fixture
def test_config():
    """Return a test configuration dictionary."""
    return {
        'api': {
            'host': 'localhost',
            'port': 8000
        },
        'embedding': {
            'default_model': 'sentence-transformers/all-MiniLM-L6-v2',
            'fallback_model': 'sentence-transformers/paraphrase-MiniLM-L3-v2',
            'dimension': 384


# Add the src directory to the Python path
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Create mock for sentence transformers before any tests run
@pytest.fixture(autouse=True)
def mock_dependencies():
    """Mock external dependencies for all tests."""
    mock_sentence_transformer = Mock()
    mock_sentence_transformer.encode.return_value = np.random.rand(1, 384).astype('float32')

    with patch.dict('sys.modules', {
        'sentence_transformers': Mock(),
        'sentence_transformers.SentenceTransformer': Mock(return_value=mock_sentence_transformer),
        'tensorflow': Mock(),
        'torch': Mock()
    }):
        yield

@pytest.fixture
def test_data_dir(tmp_path):
    """Create a temporary directory for test data."""
    data_dir = tmp_path / "test_data"
    data_dir.mkdir()
    return data_dir

@pytest.fixture
def test_config():
    """Return a test configuration dictionary."""
    return {
        'api': {
            'host': 'localhost',
            'port': 8000
        },
        'embedding': {
            'default_model': 'sentence-transformers/all-MiniLM-L6-v2',
            'fallback_model': 'sentence-transformers/paraphrase-MiniLM-L3-v2',
            'dimension': 384
        },
        'vector_store': {
            'index_type': 'flat',
            'max_batch_size': 100
        },
        'openai': {
            'model': 'gpt-4',


    """Mock external dependencies for all tests."""
    mock_sentence_transformer = Mock()
    mock_sentence_transformer.encode.return_value = np.random.rand(1, 384).astype('float32')

    with patch.dict('sys.modules', {
        'sentence_transformers': Mock(),
        'sentence_transformers.SentenceTransformer': Mock(return_value=mock_sentence_transformer),
        'tensorflow': Mock(),
        'torch': Mock()
    }):
        yield

@pytest.fixture
def test_data_dir(tmp_path):
    """Create a temporary directory for test data."""
    data_dir = tmp_path / "test_data"
    data_dir.mkdir()
    return data_dir

@pytest.fixture
def test_config():
    """Return a test configuration dictionary."""
    return {
        'api': {
            'host': 'localhost',
            'port': 8000
        },
        'embedding': {
            'default_model': 'sentence-transformers/all-MiniLM-L6-v2',
            'fallback_model': 'sentence-transformers/paraphrase-MiniLM-L3-v2',
            'dimension': 384
        },
        'vector_store': {
            'index_type': 'flat',
            'max_batch_size': 100
        },
        'openai': {
            'model': 'gpt-4',
            'temperature': 0.7,
            'max_tokens': 2000
        }
    }


import pytest
import numpy as np
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.database.vector_store import VectorStore

@pytest.fixture
def vector_store():
    """Create a real VectorStore instance."""
    return VectorStore()

def test_preprocess_text(vector_store):
    """Test text preprocessing."""
    text = "Hello, World! This is a TEST."
    words = vector_store._preprocess_text(text)
    assert words == ['hello', 'world', 'this', 'is', 'a', 'test']

def test_basic_embedding_flow(vector_store):
    """Test the basic flow of adding and searching texts."""
    # Add some texts
    texts = ["This is a test document", "Another test document"]
    metadata = [{"type": "test1"}, {"type": "test2"}]

    # Add texts to store
    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 2

    # Search for similar texts
    results = vector_store.search("test document", k=2)
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_empty_text(vector_store):
    """Test handling of empty text."""
    embedding = vector_store._get_text_embedding("")
    assert embedding.shape == (vector_store.dimension,)
    assert np.all(embedding == 0)

def test_metadata_storage(vector_store):
    """Test metadata storage and retrieval."""
    texts = ["Test document"]
    metadata = [{"key": "value"}]

    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 1
    assert vector_store.metadata[ids[0]] == metadata[0]

def test_search_with_k(vector_store):
    """Test search with different k values."""
    # Add multiple texts
    texts = ["First document", "Second document", "Third document"]
    ids = vector_store.add_texts(texts)

    # Search with different k values
    results1 = vector_store.search("document", k=1)
    results2 = vector_store.search("document", k=2)

    assert len(results1) == 1
    assert len(results2) == 2


import pytest
import numpy as np
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.database.vector_store import VectorStore

@pytest.fixture
def vector_store():
    """Create a real VectorStore instance."""
    return VectorStore()

def test_preprocess_text(vector_store):
    """Test text preprocessing."""
    text = "Hello, World! This is a TEST."
    words = vector_store._preprocess_text(text)
    assert words == ['hello', 'world', 'this', 'is', 'a', 'test']

def test_basic_embedding_flow(vector_store):
    """Test the basic flow of adding and searching texts."""
    # Add some texts
    texts = ["This is a test document", "Another test document"]
    metadata = [{"type": "test1"}, {"type": "test2"}]

    # Add texts to store
    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 2

    # Search for similar texts
    results = vector_store.search("test document", k=2)
    assert len(results) == 2


import pytest
import numpy as np
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.database.vector_store import VectorStore

@pytest.fixture
def vector_store():
    """Create a real VectorStore instance."""
    return VectorStore()

def test_preprocess_text(vector_store):
    """Test text preprocessing."""
    text = "Hello, World! This is a TEST."
    words = vector_store._preprocess_text(text)
    assert words == ['hello', 'world', 'this', 'is', 'a', 'test']

def test_basic_embedding_flow(vector_store):
    """Test the basic flow of adding and searching texts."""
    # Add some texts
    texts = ["This is a test document", "Another test document"]
    metadata = [{"type": "test1"}, {"type": "test2"}]

    # Add texts to store
    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 2

    # Search for similar texts
    results = vector_store.search("test document", k=2)
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_empty_text(vector_store):
    """Test handling of empty text."""
    embedding = vector_store._get_text_embedding("")


import pytest
import numpy as np
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.database.vector_store import VectorStore

@pytest.fixture
def vector_store():
    """Create a real VectorStore instance."""
    return VectorStore()

def test_preprocess_text(vector_store):
    """Test text preprocessing."""
    text = "Hello, World! This is a TEST."
    words = vector_store._preprocess_text(text)
    assert words == ['hello', 'world', 'this', 'is', 'a', 'test']

def test_basic_embedding_flow(vector_store):
    """Test the basic flow of adding and searching texts."""
    # Add some texts
    texts = ["This is a test document", "Another test document"]
    metadata = [{"type": "test1"}, {"type": "test2"}]

    # Add texts to store
    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 2

    # Search for similar texts
    results = vector_store.search("test document", k=2)
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_empty_text(vector_store):
    """Test handling of empty text."""
    embedding = vector_store._get_text_embedding("")
    assert embedding.shape == (vector_store.dimension,)
    assert np.all(embedding == 0)

def test_metadata_storage(vector_store):
    """Test metadata storage and retrieval."""
    texts = ["Test document"]
    metadata = [{"key": "value"}]

    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 1
    assert vector_store.metadata[ids[0]] == metadata[0]

def test_search_with_k(vector_store):
    """Test search with different k values."""
    # Add multiple texts
    texts = ["First document", "Second document", "Third document"]


    text = "Hello, World! This is a TEST."
    words = vector_store._preprocess_text(text)
    assert words == ['hello', 'world', 'this', 'is', 'a', 'test']

def test_basic_embedding_flow(vector_store):
    """Test the basic flow of adding and searching texts."""
    # Add some texts
    texts = ["This is a test document", "Another test document"]
    metadata = [{"type": "test1"}, {"type": "test2"}]

    # Add texts to store
    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 2

    # Search for similar texts
    results = vector_store.search("test document", k=2)
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_empty_text(vector_store):
    """Test handling of empty text."""
    embedding = vector_store._get_text_embedding("")
    assert embedding.shape == (vector_store.dimension,)
    assert np.all(embedding == 0)

def test_metadata_storage(vector_store):
    """Test metadata storage and retrieval."""
    texts = ["Test document"]
    metadata = [{"key": "value"}]

    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 1
    assert vector_store.metadata[ids[0]] == metadata[0]

def test_search_with_k(vector_store):
    """Test search with different k values."""
    # Add multiple texts
    texts = ["First document", "Second document", "Third document"]
    ids = vector_store.add_texts(texts)

    # Search with different k values
    results1 = vector_store.search("document", k=1)
    results2 = vector_store.search("document", k=2)


    # Add some texts
    texts = ["This is a test document", "Another test document"]
    metadata = [{"type": "test1"}, {"type": "test2"}]

    # Add texts to store
    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 2

    # Search for similar texts
    results = vector_store.search("test document", k=2)
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_empty_text(vector_store):
    """Test handling of empty text."""
    embedding = vector_store._get_text_embedding("")
    assert embedding.shape == (vector_store.dimension,)
    assert np.all(embedding == 0)

def test_metadata_storage(vector_store):
    """Test metadata storage and retrieval."""
    texts = ["Test document"]
    metadata = [{"key": "value"}]

    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 1
    assert vector_store.metadata[ids[0]] == metadata[0]

def test_search_with_k(vector_store):
    """Test search with different k values."""
    # Add multiple texts
    texts = ["First document", "Second document", "Third document"]
    ids = vector_store.add_texts(texts)

    # Search with different k values
    results1 = vector_store.search("document", k=1)
    results2 = vector_store.search("document", k=2)

    assert len(results1) == 1
    assert len(results2) == 2


    results = vector_store.search("test document", k=2)
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_empty_text(vector_store):
    """Test handling of empty text."""
    embedding = vector_store._get_text_embedding("")
    assert embedding.shape == (vector_store.dimension,)
    assert np.all(embedding == 0)

def test_metadata_storage(vector_store):
    """Test metadata storage and retrieval."""
    texts = ["Test document"]
    metadata = [{"key": "value"}]

    ids = vector_store.add_texts(texts, metadata)
    assert len(ids) == 1
    assert vector_store.metadata[ids[0]] == metadata[0]

def test_search_with_k(vector_store):
    """Test search with different k values."""
    # Add multiple texts
    texts = ["First document", "Second document", "Third document"]
    ids = vector_store.add_texts(texts)

    # Search with different k values
    results1 = vector_store.search("document", k=1)
    results2 = vector_store.search("document", k=2)

    assert len(results1) == 1
    assert len(results2) == 2


import pytest
import json
import os
from datetime import datetime
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.utils.logger import SystemLogger

@pytest.fixture
def temp_log_files(tmp_path):
    actions_log = tmp_path / "actions.log"
    thoughts_log = tmp_path / "thoughts.log"
    return str(actions_log), str(thoughts_log)

@pytest.fixture
def system_logger(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    return SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

def test_log_file_creation(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

    assert os.path.exists(actions_log)
    assert os.path.exists(thoughts_log)

def test_log_action(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action"
    test_metadata = {"key": "value"}

    system_logger.log_action(test_action, test_metadata)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_thought(system_logger, temp_log_files):
    _, thoughts_log = temp_log_files

    test_thought = "Test thought"
    test_metadata = {"key": "value"}

    system_logger.log_thought(test_thought, test_metadata)

    with open(thoughts_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "thought"
    assert log_entry["message"] == test_thought
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_without_metadata(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action without metadata"
    system_logger.log_action(test_action)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert "metadata" not in log_entry

def test_get_recent_actions(system_logger, temp_log_files):
    # Log multiple actions
    for i in range(5):
        system_logger.log_action(f"Action {i}")

    # Get recent actions
    recent_actions = system_logger.get_recent_actions(limit=3)

    assert len(recent_actions) == 3
    assert recent_actions[-1]["message"] == "Action 4"
    assert recent_actions[-2]["message"] == "Action 3"
    assert recent_actions[-3]["message"] == "Action 2"

def test_get_recent_thoughts(system_logger, temp_log_files):
    # Log multiple thoughts
    for i in range(5):
        system_logger.log_thought(f"Thought {i}")

    # Get recent thoughts
    recent_thoughts = system_logger.get_recent_thoughts(limit=3)

    assert len(recent_thoughts) == 3
    assert recent_thoughts[-1]["message"] == "Thought 4"
    assert recent_thoughts[-2]["message"] == "Thought 3"
    assert recent_thoughts[-3]["message"] == "Thought 2"

def test_timestamp_format(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    system_logger.log_action("Test timestamp")

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    # Verify timestamp is in ISO format
    timestamp = log_entry["timestamp"]
    try:
        datetime.fromisoformat(timestamp)
    except ValueError:
        pytest.fail("Timestamp is not in ISO format")

def test_handle_invalid_json_in_logs(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    # Write invalid JSON to the log file
    with open(actions_log, 'w') as f:
        f.write("Invalid JSON\n")
        f.write(json.dumps({"type": "action", "message": "Valid entry"}) + "\n")

    # Should skip invalid JSON and return valid entries
    recent_actions = system_logger.get_recent_actions()

    assert len(recent_actions) == 1
    assert recent_actions[0]["message"] == "Valid entry"


import pytest
import json
import os
from datetime import datetime
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.utils.logger import SystemLogger

@pytest.fixture
def temp_log_files(tmp_path):
    actions_log = tmp_path / "actions.log"
    thoughts_log = tmp_path / "thoughts.log"
    return str(actions_log), str(thoughts_log)

@pytest.fixture
def system_logger(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    return SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

def test_log_file_creation(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

    assert os.path.exists(actions_log)
    assert os.path.exists(thoughts_log)

def test_log_action(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action"
    test_metadata = {"key": "value"}

    system_logger.log_action(test_action, test_metadata)


import pytest
import json
import os
from datetime import datetime
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.utils.logger import SystemLogger

@pytest.fixture
def temp_log_files(tmp_path):
    actions_log = tmp_path / "actions.log"
    thoughts_log = tmp_path / "thoughts.log"
    return str(actions_log), str(thoughts_log)

@pytest.fixture
def system_logger(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    return SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

def test_log_file_creation(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

    assert os.path.exists(actions_log)
    assert os.path.exists(thoughts_log)

def test_log_action(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action"
    test_metadata = {"key": "value"}

    system_logger.log_action(test_action, test_metadata)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"


import pytest
import json
import os
from datetime import datetime
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.utils.logger import SystemLogger

@pytest.fixture
def temp_log_files(tmp_path):
    actions_log = tmp_path / "actions.log"
    thoughts_log = tmp_path / "thoughts.log"
    return str(actions_log), str(thoughts_log)

@pytest.fixture
def system_logger(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    return SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

def test_log_file_creation(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

    assert os.path.exists(actions_log)
    assert os.path.exists(thoughts_log)

def test_log_action(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action"
    test_metadata = {"key": "value"}

    system_logger.log_action(test_action, test_metadata)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_thought(system_logger, temp_log_files):
    _, thoughts_log = temp_log_files


from src.utils.logger import SystemLogger

@pytest.fixture
def temp_log_files(tmp_path):
    actions_log = tmp_path / "actions.log"
    thoughts_log = tmp_path / "thoughts.log"
    return str(actions_log), str(thoughts_log)

@pytest.fixture
def system_logger(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    return SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

def test_log_file_creation(temp_log_files):
    actions_log, thoughts_log = temp_log_files
    SystemLogger(actions_log_path=actions_log, thoughts_log_path=thoughts_log)

    assert os.path.exists(actions_log)
    assert os.path.exists(thoughts_log)

def test_log_action(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action"
    test_metadata = {"key": "value"}

    system_logger.log_action(test_action, test_metadata)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_thought(system_logger, temp_log_files):
    _, thoughts_log = temp_log_files

    test_thought = "Test thought"
    test_metadata = {"key": "value"}

    system_logger.log_thought(test_thought, test_metadata)

    with open(thoughts_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "thought"
    assert log_entry["message"] == test_thought
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_without_metadata(system_logger, temp_log_files):
    actions_log, _ = temp_log_files


    assert os.path.exists(actions_log)
    assert os.path.exists(thoughts_log)

def test_log_action(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action"
    test_metadata = {"key": "value"}

    system_logger.log_action(test_action, test_metadata)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_thought(system_logger, temp_log_files):
    _, thoughts_log = temp_log_files

    test_thought = "Test thought"
    test_metadata = {"key": "value"}

    system_logger.log_thought(test_thought, test_metadata)

    with open(thoughts_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "thought"
    assert log_entry["message"] == test_thought
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_without_metadata(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action without metadata"
    system_logger.log_action(test_action)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert "metadata" not in log_entry

def test_get_recent_actions(system_logger, temp_log_files):
    # Log multiple actions
    for i in range(5):
        system_logger.log_action(f"Action {i}")

    # Get recent actions


    assert log_entry["message"] == test_action
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_thought(system_logger, temp_log_files):
    _, thoughts_log = temp_log_files

    test_thought = "Test thought"
    test_metadata = {"key": "value"}

    system_logger.log_thought(test_thought, test_metadata)

    with open(thoughts_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "thought"
    assert log_entry["message"] == test_thought
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_without_metadata(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action without metadata"
    system_logger.log_action(test_action)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert "metadata" not in log_entry

def test_get_recent_actions(system_logger, temp_log_files):
    # Log multiple actions
    for i in range(5):
        system_logger.log_action(f"Action {i}")

    # Get recent actions
    recent_actions = system_logger.get_recent_actions(limit=3)

    assert len(recent_actions) == 3
    assert recent_actions[-1]["message"] == "Action 4"
    assert recent_actions[-2]["message"] == "Action 3"
    assert recent_actions[-3]["message"] == "Action 2"

def test_get_recent_thoughts(system_logger, temp_log_files):
    # Log multiple thoughts
    for i in range(5):
        system_logger.log_thought(f"Thought {i}")

    # Get recent thoughts


        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "thought"
    assert log_entry["message"] == test_thought
    assert log_entry["metadata"] == test_metadata
    assert "timestamp" in log_entry

def test_log_without_metadata(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    test_action = "Test action without metadata"
    system_logger.log_action(test_action)

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert "metadata" not in log_entry

def test_get_recent_actions(system_logger, temp_log_files):
    # Log multiple actions
    for i in range(5):
        system_logger.log_action(f"Action {i}")

    # Get recent actions
    recent_actions = system_logger.get_recent_actions(limit=3)

    assert len(recent_actions) == 3
    assert recent_actions[-1]["message"] == "Action 4"
    assert recent_actions[-2]["message"] == "Action 3"
    assert recent_actions[-3]["message"] == "Action 2"

def test_get_recent_thoughts(system_logger, temp_log_files):
    # Log multiple thoughts
    for i in range(5):
        system_logger.log_thought(f"Thought {i}")

    # Get recent thoughts
    recent_thoughts = system_logger.get_recent_thoughts(limit=3)

    assert len(recent_thoughts) == 3
    assert recent_thoughts[-1]["message"] == "Thought 4"
    assert recent_thoughts[-2]["message"] == "Thought 3"
    assert recent_thoughts[-3]["message"] == "Thought 2"

def test_timestamp_format(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    system_logger.log_action("Test timestamp")

    with open(actions_log, 'r') as f:


    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    assert log_entry["type"] == "action"
    assert log_entry["message"] == test_action
    assert "metadata" not in log_entry

def test_get_recent_actions(system_logger, temp_log_files):
    # Log multiple actions
    for i in range(5):
        system_logger.log_action(f"Action {i}")

    # Get recent actions
    recent_actions = system_logger.get_recent_actions(limit=3)

    assert len(recent_actions) == 3
    assert recent_actions[-1]["message"] == "Action 4"
    assert recent_actions[-2]["message"] == "Action 3"
    assert recent_actions[-3]["message"] == "Action 2"

def test_get_recent_thoughts(system_logger, temp_log_files):
    # Log multiple thoughts
    for i in range(5):
        system_logger.log_thought(f"Thought {i}")

    # Get recent thoughts
    recent_thoughts = system_logger.get_recent_thoughts(limit=3)

    assert len(recent_thoughts) == 3
    assert recent_thoughts[-1]["message"] == "Thought 4"
    assert recent_thoughts[-2]["message"] == "Thought 3"
    assert recent_thoughts[-3]["message"] == "Thought 2"

def test_timestamp_format(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    system_logger.log_action("Test timestamp")

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    # Verify timestamp is in ISO format
    timestamp = log_entry["timestamp"]
    try:
        datetime.fromisoformat(timestamp)
    except ValueError:
        pytest.fail("Timestamp is not in ISO format")

def test_handle_invalid_json_in_logs(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    # Write invalid JSON to the log file


    recent_actions = system_logger.get_recent_actions(limit=3)

    assert len(recent_actions) == 3
    assert recent_actions[-1]["message"] == "Action 4"
    assert recent_actions[-2]["message"] == "Action 3"
    assert recent_actions[-3]["message"] == "Action 2"

def test_get_recent_thoughts(system_logger, temp_log_files):
    # Log multiple thoughts
    for i in range(5):
        system_logger.log_thought(f"Thought {i}")

    # Get recent thoughts
    recent_thoughts = system_logger.get_recent_thoughts(limit=3)

    assert len(recent_thoughts) == 3
    assert recent_thoughts[-1]["message"] == "Thought 4"
    assert recent_thoughts[-2]["message"] == "Thought 3"
    assert recent_thoughts[-3]["message"] == "Thought 2"

def test_timestamp_format(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    system_logger.log_action("Test timestamp")

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    # Verify timestamp is in ISO format
    timestamp = log_entry["timestamp"]
    try:
        datetime.fromisoformat(timestamp)
    except ValueError:
        pytest.fail("Timestamp is not in ISO format")

def test_handle_invalid_json_in_logs(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    # Write invalid JSON to the log file
    with open(actions_log, 'w') as f:
        f.write("Invalid JSON\n")
        f.write(json.dumps({"type": "action", "message": "Valid entry"}) + "\n")

    # Should skip invalid JSON and return valid entries
    recent_actions = system_logger.get_recent_actions()

    assert len(recent_actions) == 1
    assert recent_actions[0]["message"] == "Valid entry"


    assert len(recent_thoughts) == 3
    assert recent_thoughts[-1]["message"] == "Thought 4"
    assert recent_thoughts[-2]["message"] == "Thought 3"
    assert recent_thoughts[-3]["message"] == "Thought 2"

def test_timestamp_format(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    system_logger.log_action("Test timestamp")

    with open(actions_log, 'r') as f:
        log_entry = json.loads(f.read().strip())

    # Verify timestamp is in ISO format
    timestamp = log_entry["timestamp"]
    try:
        datetime.fromisoformat(timestamp)
    except ValueError:
        pytest.fail("Timestamp is not in ISO format")

def test_handle_invalid_json_in_logs(system_logger, temp_log_files):
    actions_log, _ = temp_log_files

    # Write invalid JSON to the log file
    with open(actions_log, 'w') as f:
        f.write("Invalid JSON\n")
        f.write(json.dumps({"type": "action", "message": "Valid entry"}) + "\n")

    # Should skip invalid JSON and return valid entries
    recent_actions = system_logger.get_recent_actions()

    assert len(recent_actions) == 1
    assert recent_actions[0]["message"] == "Valid entry"


import pytest
from fastapi.testclient import TestClient
import os
import sys
from dotenv import load_dotenv

# Load environment variables before importing app
load_dotenv()

# Ensure we have an API key for tests
if not os.getenv('OPENAI_API_KEY'):
    pytest.skip("Skipping tests that require OpenAI API key", allow_module_level=True)

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.api.main import app

@pytest.fixture
def client():
    """Create a test client."""
    return TestClient(app)

def test_health_check(client):
    """Test the health check endpoint."""
    response = client.get("/health")
    assert response.status_code == 200
    assert response.json() == {"status": "healthy"}

def test_add_and_search_flow(client):
    """Test the basic flow of adding texts and searching."""
    # First add some texts
    add_response = client.post(
        "/add-texts",
        json={
            "texts": ["This is a test document", "Another test document"],
            "metadata": [{"type": "test1"}, {"type": "test2"}]
        }
    )
    assert add_response.status_code == 200
    assert "ids" in add_response.json()

    # Then search
    search_response = client.post(
        "/search",
        json={
            "query": "test document",
            "k": 2
        }
    )
    assert search_response.status_code == 200
    results = search_response.json()["results"]
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_invalid_requests(client):
    """Test handling of invalid requests."""
    # Missing required field
    response1 = client.post("/add-texts", json={})
    assert response1.status_code == 422

    # Invalid k value
    response2 = client.post("/search", json={"query": "test", "k": -1})
    assert response2.status_code == 422

    # Empty query
    response3 = client.post("/search", json={"query": "", "k": 1})
    assert response3.status_code == 200  # Should handle empty query gracefully


client

import pytest
from fastapi.testclient import TestClient
import os
import sys
from dotenv import load_dotenv

# Load environment variables before importing app
load_dotenv()

# Ensure we have an API key for tests
if not os.getenv('OPENAI_API_KEY'):
    pytest.skip("Skipping tests that require OpenAI API key", allow_module_level=True)

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.api.main import app

@pytest.fixture
def client():
    """Create a test client."""
    return TestClient(app)

def test_health_check(client):
    """Test the health check endpoint."""
    response = client.get("/health")
    assert response.status_code == 200
    assert response.json() == {"status": "healthy"}

def test_add_and_search_flow(client):
    """Test the basic flow of adding texts and searching."""
    # First add some texts
    add_response = client.post(
        "/add-texts",
        json={
            "texts": ["This is a test document", "Another test document"],
            "metadata": [{"type": "test1"}, {"type": "test2"}]
        }
    )
    assert add_response.status_code == 200
    assert "ids" in add_response.json()


from fastapi.testclient import TestClient
import os
import sys
from dotenv import load_dotenv

# Load environment variables before importing app
load_dotenv()

# Ensure we have an API key for tests
if not os.getenv('OPENAI_API_KEY'):
    pytest.skip("Skipping tests that require OpenAI API key", allow_module_level=True)

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.api.main import app

@pytest.fixture
def client():
    """Create a test client."""
    return TestClient(app)

def test_health_check(client):
    """Test the health check endpoint."""
    response = client.get("/health")
    assert response.status_code == 200
    assert response.json() == {"status": "healthy"}

def test_add_and_search_flow(client):
    """Test the basic flow of adding texts and searching."""
    # First add some texts
    add_response = client.post(
        "/add-texts",
        json={
            "texts": ["This is a test document", "Another test document"],
            "metadata": [{"type": "test1"}, {"type": "test2"}]
        }
    )
    assert add_response.status_code == 200
    assert "ids" in add_response.json()

    # Then search
    search_response = client.post(
        "/search",
        json={
            "query": "test document",
            "k": 2


load_dotenv()

# Ensure we have an API key for tests
if not os.getenv('OPENAI_API_KEY'):
    pytest.skip("Skipping tests that require OpenAI API key", allow_module_level=True)

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.api.main import app

@pytest.fixture
def client():
    """Create a test client."""
    return TestClient(app)

def test_health_check(client):
    """Test the health check endpoint."""
    response = client.get("/health")
    assert response.status_code == 200
    assert response.json() == {"status": "healthy"}

def test_add_and_search_flow(client):
    """Test the basic flow of adding texts and searching."""
    # First add some texts
    add_response = client.post(
        "/add-texts",
        json={
            "texts": ["This is a test document", "Another test document"],
            "metadata": [{"type": "test1"}, {"type": "test2"}]
        }
    )
    assert add_response.status_code == 200
    assert "ids" in add_response.json()

    # Then search
    search_response = client.post(
        "/search",
        json={
            "query": "test document",
            "k": 2
        }
    )
    assert search_response.status_code == 200
    results = search_response.json()["results"]
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_invalid_requests(client):
    """Test handling of invalid requests."""
    # Missing required field
    response1 = client.post("/add-texts", json={})
    assert response1.status_code == 422

    # Invalid k value
    response2 = client.post("/search", json={"query": "test", "k": -1})
    assert response2.status_code == 422

    # Empty query
    response3 = client.post("/search", json={"query": "", "k": 1})
    assert response3.status_code == 200  # Should handle empty query gracefully


            "metadata": [{"type": "test1"}, {"type": "test2"}]
        }
    )
    assert add_response.status_code == 200
    assert "ids" in add_response.json()

    # Then search
    search_response = client.post(
        "/search",
        json={
            "query": "test document",
            "k": 2
        }
    )
    assert search_response.status_code == 200
    results = search_response.json()["results"]
    assert len(results) == 2
    assert all('score' in r for r in results)
    assert all('metadata' in r for r in results)

def test_invalid_requests(client):
    """Test handling of invalid requests."""
    # Missing required field
    response1 = client.post("/add-texts", json={})
    assert response1.status_code == 422

    # Invalid k value
    response2 = client.post("/search", json={"query": "test", "k": -1})
    assert response2.status_code == 422

    # Empty query
    response3 = client.post("/search", json={"query": "", "k": 1})
    assert response3.status_code == 200  # Should handle empty query gracefully


import pytest
from unittest.mock import patch, MagicMock
import json
import os
import sys
from dotenv import load_dotenv

# Load environment variables from .env
load_dotenv()

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.chatgpt.knowledge_processor import KnowledgeProcessor

@pytest.fixture
def mock_openai():
    """Create a mock OpenAI client."""
    mock_response = {
        "choices": [{
            "message": {
                "content": json.dumps({
                    "core_knowledge": [
                        {
                            "title": "Test Knowledge",
                            "description": "Test Description"
                        }
                    ],
                    "tools": [
                        {
                            "name": "test_tool",
                            "description": "Tool Description"
                        }
                    ],
                    "examples": [
                        {
                            "query": "Test Query",
                            "expected_response": "Test Response"
                        }
                    ]
                })
            }
        }]
    }

    # Create a proper mock response
    mock_completion = MagicMock()
    mock_completion.choices = [
        MagicMock(
            message=MagicMock(
                content=mock_response["choices"][0]["message"]["content"]
            )
        )
    ]

    # Set up the mock client
    mock_instance = MagicMock()
    mock_instance.chat.completions.create.return_value = mock_completion

    with patch('openai.OpenAI', return_value=mock_instance):
        yield mock_instance

@pytest.fixture
def knowledge_processor(mock_openai):
    """Create a KnowledgeProcessor instance with mocked OpenAI client."""
    # Use the API key from environment instead of test-key
    return KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

def test_process_user_input(knowledge_processor):
    """Test basic knowledge processing."""
    # Process input
    result = knowledge_processor.process_user_input("Test input")

    # Verify structure
    assert "structured_knowledge" in result
    assert "text_chunks" in result
    assert "metadata" in result

    # Verify content
    knowledge = result["structured_knowledge"]
    assert "core_knowledge" in knowledge
    assert "tools" in knowledge
    assert "examples" in knowledge

def test_generate_text_chunks(knowledge_processor):
    """Test text chunk generation."""
    test_knowledge = {
        "core_knowledge": [
            {
                "title": "Test",
                "description": "Description"
            }
        ],
        "tools": [
            {
                "name": "tool",
                "description": "Tool desc"
            }
        ],
        "examples": []
    }

    chunks = knowledge_processor._generate_text_chunks(test_knowledge)
    assert len(chunks) == 2  # One for knowledge, one for tool
    assert any("Test" in chunk for chunk in chunks)
    assert any("tool" in chunk for chunk in chunks)


import pytest
from unittest.mock import patch, MagicMock
import json
import os
import sys
from dotenv import load_dotenv

# Load environment variables from .env
load_dotenv()

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.chatgpt.knowledge_processor import KnowledgeProcessor

@pytest.fixture
def mock_openai():
    """Create a mock OpenAI client."""
    mock_response = {
        "choices": [{
            "message": {
                "content": json.dumps({
                    "core_knowledge": [
                        {
                            "title": "Test Knowledge",
                            "description": "Test Description"
                        }
                    ],
                    "tools": [
                        {
                            "name": "test_tool",
                            "description": "Tool Description"
                        }
                    ],
                    "examples": [
                        {
                            "query": "Test Query",
                            "expected_response": "Test Response"
                        }
                    ]
                })
            }
        }]
    }

    # Create a proper mock response
    mock_completion = MagicMock()
    mock_completion.choices = [
        MagicMock(
            message=MagicMock(
                content=mock_response["choices"][0]["message"]["content"]
            )
        )
    ]

    # Set up the mock client
    mock_instance = MagicMock()
    mock_instance.chat.completions.create.return_value = mock_completion

    with patch('openai.OpenAI', return_value=mock_instance):
        yield mock_instance

@pytest.fixture
def knowledge_processor(mock_openai):
    """Create a KnowledgeProcessor instance with mocked OpenAI client."""
    # Use the API key from environment instead of test-key
    return KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

def test_process_user_input(knowledge_processor):
    """Test basic knowledge processing."""
    # Process input
    result = knowledge_processor.process_user_input("Test input")

    # Verify structure
    assert "structured_knowledge" in result
    assert "text_chunks" in result
    assert "metadata" in result

    # Verify content
    knowledge = result["structured_knowledge"]
    assert "core_knowledge" in knowledge


    }

    # Create a proper mock response
    mock_completion = MagicMock()
    mock_completion.choices = [
        MagicMock(
            message=MagicMock(
                content=mock_response["choices"][0]["message"]["content"]
            )
        )
    ]

    # Set up the mock client
    mock_instance = MagicMock()
    mock_instance.chat.completions.create.return_value = mock_completion

    with patch('openai.OpenAI', return_value=mock_instance):
        yield mock_instance

@pytest.fixture
def knowledge_processor(mock_openai):
    """Create a KnowledgeProcessor instance with mocked OpenAI client."""
    # Use the API key from environment instead of test-key
    return KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

def test_process_user_input(knowledge_processor):
    """Test basic knowledge processing."""
    # Process input
    result = knowledge_processor.process_user_input("Test input")

    # Verify structure
    assert "structured_knowledge" in result
    assert "text_chunks" in result
    assert "metadata" in result

    # Verify content
    knowledge = result["structured_knowledge"]
    assert "core_knowledge" in knowledge
    assert "tools" in knowledge
    assert "examples" in knowledge

def test_generate_text_chunks(knowledge_processor):
    """Test text chunk generation."""
    test_knowledge = {


        MagicMock(
            message=MagicMock(
                content=mock_response["choices"][0]["message"]["content"]
            )
        )
    ]

    # Set up the mock client
    mock_instance = MagicMock()
    mock_instance.chat.completions.create.return_value = mock_completion

    with patch('openai.OpenAI', return_value=mock_instance):
        yield mock_instance

@pytest.fixture
def knowledge_processor(mock_openai):
    """Create a KnowledgeProcessor instance with mocked OpenAI client."""
    # Use the API key from environment instead of test-key
    return KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

def test_process_user_input(knowledge_processor):
    """Test basic knowledge processing."""
    # Process input
    result = knowledge_processor.process_user_input("Test input")

    # Verify structure
    assert "structured_knowledge" in result
    assert "text_chunks" in result
    assert "metadata" in result

    # Verify content
    knowledge = result["structured_knowledge"]
    assert "core_knowledge" in knowledge
    assert "tools" in knowledge
    assert "examples" in knowledge

def test_generate_text_chunks(knowledge_processor):
    """Test text chunk generation."""
    test_knowledge = {
        "core_knowledge": [
            {
                "title": "Test",
                "description": "Description"
            }
        ],
        "tools": [
            {
                "name": "tool",
                "description": "Tool desc"
            }
        ],
        "examples": []
    }

    chunks = knowledge_processor._generate_text_chunks(test_knowledge)


    """Create a KnowledgeProcessor instance with mocked OpenAI client."""
    # Use the API key from environment instead of test-key
    return KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

def test_process_user_input(knowledge_processor):
    """Test basic knowledge processing."""
    # Process input
    result = knowledge_processor.process_user_input("Test input")

    # Verify structure
    assert "structured_knowledge" in result
    assert "text_chunks" in result
    assert "metadata" in result

    # Verify content
    knowledge = result["structured_knowledge"]
    assert "core_knowledge" in knowledge
    assert "tools" in knowledge
    assert "examples" in knowledge

def test_generate_text_chunks(knowledge_processor):
    """Test text chunk generation."""
    test_knowledge = {
        "core_knowledge": [
            {
                "title": "Test",
                "description": "Description"
            }
        ],
        "tools": [
            {
                "name": "tool",
                "description": "Tool desc"
            }
        ],
        "examples": []
    }

    chunks = knowledge_processor._generate_text_chunks(test_knowledge)
    assert len(chunks) == 2  # One for knowledge, one for tool
    assert any("Test" in chunk for chunk in chunks)
    assert any("tool" in chunk for chunk in chunks)


from typing import List, Dict, Any, Optional
import faiss
import numpy as np
from loguru import logger
import yaml
import os
import re
from collections import Counter

class VectorStore:
    def __init__(self, config_path: str = "config.yaml"):
        with open(config_path, 'r') as f:
            self.config = yaml.safe_load(f)

        self.index = None
        self.metadata = {}
        self.dimension = 100  # Fixed dimension for our simple embeddings
        self.vocabulary = {}  # Word to index mapping
        self.next_word_index = 0

    def _preprocess_text(self, text: str) -> List[str]:
        """Simple text preprocessing."""
        # Convert to lowercase and split into words
        words = re.findall(r'\w+', text.lower())
        return words

    def _get_word_index(self, word: str) -> int:
        """Get or create index for a word."""
        if word not in self.vocabulary:
            self.vocabulary[word] = self.next_word_index
            self.next_word_index += 1
        return self.vocabulary[word]

    def _get_text_embedding(self, text: str) -> np.ndarray:
        """Create a simple frequency-based embedding for text."""
        words = self._preprocess_text(text)
        if not words:
            return np.zeros(self.dimension)

        # Count word frequencies
        word_counts = Counter(words)

        # Create embedding vector
        embedding = np.zeros(self.dimension)
        for word, count in word_counts.items():
            index = self._get_word_index(word) % self.dimension
            embedding[index] += count

        # Normalize the vector
        norm = np.linalg.norm(embedding)
        if norm > 0:
            embedding = embedding / norm

        return embedding

    def create_index(self, index_type: str = "flat") -> None:
        """Initialize FAISS index."""
        if index_type == "flat":
            self.index = faiss.IndexFlatL2(self.dimension)
        elif index_type == "hnsw":
            self.index = faiss.IndexHNSWFlat(self.dimension, 32)
        else:
            raise ValueError(f"Unsupported index type: {index_type}")

        logger.info(f"Created {index_type} index with dimension {self.dimension}")

    def add_texts(self, texts: List[str], metadata: Optional[List[Dict]] = None) -> List[int]:
        """Add texts to the vector store."""
        if self.index is None:
            self.create_index()

        # Get embeddings for all texts
        embeddings = []
        for text in texts:
            embedding = self._get_text_embedding(text)
            embeddings.append(embedding)

        embeddings_array = np.array(embeddings).astype('float32')

        if metadata:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))
            for i, meta in zip(ids, metadata):
                self.metadata[i] = meta
        else:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))

        self.index.add(embeddings_array)
        return ids

    def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
        """Search the vector store for similar texts."""
        query_embedding = self._get_text_embedding(query)

        D, I = self.index.search(
            np.array([query_embedding]).astype('float32'),
            k
        )

        results = []
        for score, idx in zip(D[0], I[0]):
            result = {
                'id': int(idx),
                'score': float(score),
                'metadata': self.metadata.get(int(idx), {})
            }
            results.append(result)

        return results


from typing import List, Dict, Any, Optional
import faiss
import numpy as np
from loguru import logger
import yaml
import os
import re
from collections import Counter

class VectorStore:
    def __init__(self, config_path: str = "config.yaml"):
        with open(config_path, 'r') as f:
            self.config = yaml.safe_load(f)

        self.index = None
        self.metadata = {}
        self.dimension = 100  # Fixed dimension for our simple embeddings
        self.vocabulary = {}  # Word to index mapping
        self.next_word_index = 0

    def _preprocess_text(self, text: str) -> List[str]:
        """Simple text preprocessing."""
        # Convert to lowercase and split into words
        words = re.findall(r'\w+', text.lower())
        return words

    def _get_word_index(self, word: str) -> int:
        """Get or create index for a word."""
        if word not in self.vocabulary:
            self.vocabulary[word] = self.next_word_index
            self.next_word_index += 1
        return self.vocabulary[word]

    def _get_text_embedding(self, text: str) -> np.ndarray:
        """Create a simple frequency-based embedding for text."""
        words = self._preprocess_text(text)
        if not words:
            return np.zeros(self.dimension)


from typing import List, Dict, Any, Optional
import faiss
import numpy as np
from loguru import logger
import yaml
import os
import re
from collections import Counter

class VectorStore:
    def __init__(self, config_path: str = "config.yaml"):
        with open(config_path, 'r') as f:
            self.config = yaml.safe_load(f)

        self.index = None
        self.metadata = {}
        self.dimension = 100  # Fixed dimension for our simple embeddings
        self.vocabulary = {}  # Word to index mapping
        self.next_word_index = 0

    def _preprocess_text(self, text: str) -> List[str]:
        """Simple text preprocessing."""
        # Convert to lowercase and split into words
        words = re.findall(r'\w+', text.lower())
        return words

    def _get_word_index(self, word: str) -> int:
        """Get or create index for a word."""
        if word not in self.vocabulary:
            self.vocabulary[word] = self.next_word_index
            self.next_word_index += 1
        return self.vocabulary[word]

    def _get_text_embedding(self, text: str) -> np.ndarray:
        """Create a simple frequency-based embedding for text."""
        words = self._preprocess_text(text)
        if not words:
            return np.zeros(self.dimension)

        # Count word frequencies
        word_counts = Counter(words)

        # Create embedding vector
        embedding = np.zeros(self.dimension)
        for word, count in word_counts.items():


import re
from collections import Counter

class VectorStore:
    def __init__(self, config_path: str = "config.yaml"):
        with open(config_path, 'r') as f:
            self.config = yaml.safe_load(f)

        self.index = None
        self.metadata = {}
        self.dimension = 100  # Fixed dimension for our simple embeddings
        self.vocabulary = {}  # Word to index mapping
        self.next_word_index = 0

    def _preprocess_text(self, text: str) -> List[str]:
        """Simple text preprocessing."""
        # Convert to lowercase and split into words
        words = re.findall(r'\w+', text.lower())
        return words

    def _get_word_index(self, word: str) -> int:
        """Get or create index for a word."""
        if word not in self.vocabulary:
            self.vocabulary[word] = self.next_word_index
            self.next_word_index += 1
        return self.vocabulary[word]

    def _get_text_embedding(self, text: str) -> np.ndarray:
        """Create a simple frequency-based embedding for text."""
        words = self._preprocess_text(text)
        if not words:
            return np.zeros(self.dimension)

        # Count word frequencies
        word_counts = Counter(words)

        # Create embedding vector
        embedding = np.zeros(self.dimension)
        for word, count in word_counts.items():
            index = self._get_word_index(word) % self.dimension
            embedding[index] += count

        # Normalize the vector
        norm = np.linalg.norm(embedding)
        if norm > 0:
            embedding = embedding / norm


        self.index = None
        self.metadata = {}
        self.dimension = 100  # Fixed dimension for our simple embeddings
        self.vocabulary = {}  # Word to index mapping
        self.next_word_index = 0

    def _preprocess_text(self, text: str) -> List[str]:
        """Simple text preprocessing."""
        # Convert to lowercase and split into words
        words = re.findall(r'\w+', text.lower())
        return words

    def _get_word_index(self, word: str) -> int:
        """Get or create index for a word."""
        if word not in self.vocabulary:
            self.vocabulary[word] = self.next_word_index
            self.next_word_index += 1
        return self.vocabulary[word]

    def _get_text_embedding(self, text: str) -> np.ndarray:
        """Create a simple frequency-based embedding for text."""
        words = self._preprocess_text(text)
        if not words:
            return np.zeros(self.dimension)

        # Count word frequencies
        word_counts = Counter(words)

        # Create embedding vector
        embedding = np.zeros(self.dimension)
        for word, count in word_counts.items():
            index = self._get_word_index(word) % self.dimension
            embedding[index] += count

        # Normalize the vector
        norm = np.linalg.norm(embedding)
        if norm > 0:
            embedding = embedding / norm

        return embedding

    def create_index(self, index_type: str = "flat") -> None:
        """Initialize FAISS index."""
        if index_type == "flat":
            self.index = faiss.IndexFlatL2(self.dimension)
        elif index_type == "hnsw":
            self.index = faiss.IndexHNSWFlat(self.dimension, 32)
        else:
            raise ValueError(f"Unsupported index type: {index_type}")

        logger.info(f"Created {index_type} index with dimension {self.dimension}")

    def add_texts(self, texts: List[str], metadata: Optional[List[Dict]] = None) -> List[int]:
        """Add texts to the vector store."""
        if self.index is None:
            self.create_index()

        # Get embeddings for all texts
        embeddings = []
        for text in texts:


        words = self._preprocess_text(text)
        if not words:
            return np.zeros(self.dimension)

        # Count word frequencies
        word_counts = Counter(words)

        # Create embedding vector
        embedding = np.zeros(self.dimension)
        for word, count in word_counts.items():
            index = self._get_word_index(word) % self.dimension
            embedding[index] += count

        # Normalize the vector
        norm = np.linalg.norm(embedding)
        if norm > 0:
            embedding = embedding / norm

        return embedding

    def create_index(self, index_type: str = "flat") -> None:
        """Initialize FAISS index."""
        if index_type == "flat":
            self.index = faiss.IndexFlatL2(self.dimension)
        elif index_type == "hnsw":
            self.index = faiss.IndexHNSWFlat(self.dimension, 32)
        else:
            raise ValueError(f"Unsupported index type: {index_type}")

        logger.info(f"Created {index_type} index with dimension {self.dimension}")

    def add_texts(self, texts: List[str], metadata: Optional[List[Dict]] = None) -> List[int]:
        """Add texts to the vector store."""
        if self.index is None:
            self.create_index()

        # Get embeddings for all texts
        embeddings = []
        for text in texts:
            embedding = self._get_text_embedding(text)
            embeddings.append(embedding)

        embeddings_array = np.array(embeddings).astype('float32')

        if metadata:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))
            for i, meta in zip(ids, metadata):
                self.metadata[i] = meta
        else:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))


            embedding[index] += count

        # Normalize the vector
        norm = np.linalg.norm(embedding)
        if norm > 0:
            embedding = embedding / norm

        return embedding

    def create_index(self, index_type: str = "flat") -> None:
        """Initialize FAISS index."""
        if index_type == "flat":
            self.index = faiss.IndexFlatL2(self.dimension)
        elif index_type == "hnsw":
            self.index = faiss.IndexHNSWFlat(self.dimension, 32)
        else:
            raise ValueError(f"Unsupported index type: {index_type}")

        logger.info(f"Created {index_type} index with dimension {self.dimension}")

    def add_texts(self, texts: List[str], metadata: Optional[List[Dict]] = None) -> List[int]:
        """Add texts to the vector store."""
        if self.index is None:
            self.create_index()

        # Get embeddings for all texts
        embeddings = []
        for text in texts:
            embedding = self._get_text_embedding(text)
            embeddings.append(embedding)

        embeddings_array = np.array(embeddings).astype('float32')

        if metadata:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))
            for i, meta in zip(ids, metadata):
                self.metadata[i] = meta
        else:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))

        self.index.add(embeddings_array)
        return ids

    def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
        """Search the vector store for similar texts."""
        query_embedding = self._get_text_embedding(query)

        D, I = self.index.search(
            np.array([query_embedding]).astype('float32'),
            k
        )

        results = []
        for score, idx in zip(D[0], I[0]):
            result = {
                'id': int(idx),
                'score': float(score),
                'metadata': self.metadata.get(int(idx), {})
            }
            results.append(result)

        return results


search

            self.create_index()

        # Get embeddings for all texts
        embeddings = []
        for text in texts:
            embedding = self._get_text_embedding(text)
            embeddings.append(embedding)

        embeddings_array = np.array(embeddings).astype('float32')

        if metadata:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))
            for i, meta in zip(ids, metadata):
                self.metadata[i] = meta
        else:
            ids = list(range(len(self.metadata), len(self.metadata) + len(texts)))

        self.index.add(embeddings_array)
        return ids

    def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
        """Search the vector store for similar texts."""
        query_embedding = self._get_text_embedding(query)

        D, I = self.index.search(
            np.array([query_embedding]).astype('float32'),
            k
        )

        results = []
        for score, idx in zip(D[0], I[0]):
            result = {
                'id': int(idx),
                'score': float(score),
                'metadata': self.metadata.get(int(idx), {})
            }
            results.append(result)

        return results


from typing import List, Dict, Any, Tuple
import pandas as pd
import json
from loguru import logger

class DataProcessor:
    @staticmethod
    def process_file(file_path: str, file_format: str) -> Tuple[List[str], List[Dict]]:
        """Process input file and return texts and metadata."""
        try:
            if file_format == 'csv':
                return DataProcessor._process_csv(file_path)
            elif file_format == 'json':
                return DataProcessor._process_json(file_path)
            elif file_format == 'txt':
                return DataProcessor._process_txt(file_path)
            else:
                raise ValueError(f"Unsupported file format: {file_format}")
        except Exception as e:
            logger.error(f"Error processing file: {e}")
            raise

    @staticmethod
    def _process_csv(file_path: str) -> Tuple[List[str], List[Dict]]:
        df = pd.read_csv(file_path)
        texts = df.iloc[:, 0].tolist()  # Assume first column contains text
        metadata = df.to_dict('records')
        return texts, metadata

    @staticmethod
    def _process_json(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            data = json.load(f)
        texts = [item.get('text', '') for item in data]
        metadata = data
        return texts, metadata

    @staticmethod
    def _process_txt(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            texts = f.readlines()
        metadata = [{'line_number': i} for i in range(len(texts))]
        return texts, metadata


from typing import List, Dict, Any, Tuple
import pandas as pd
import json
from loguru import logger

class DataProcessor:
    @staticmethod
    def process_file(file_path: str, file_format: str) -> Tuple[List[str], List[Dict]]:
        """Process input file and return texts and metadata."""
        try:
            if file_format == 'csv':
                return DataProcessor._process_csv(file_path)
            elif file_format == 'json':
                return DataProcessor._process_json(file_path)
            elif file_format == 'txt':
                return DataProcessor._process_txt(file_path)
            else:
                raise ValueError(f"Unsupported file format: {file_format}")
        except Exception as e:
            logger.error(f"Error processing file: {e}")
            raise

    @staticmethod
    def _process_csv(file_path: str) -> Tuple[List[str], List[Dict]]:
        df = pd.read_csv(file_path)
        texts = df.iloc[:, 0].tolist()  # Assume first column contains text
        metadata = df.to_dict('records')
        return texts, metadata

    @staticmethod
    def _process_json(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            data = json.load(f)
        texts = [item.get('text', '') for item in data]
        metadata = data
        return texts, metadata

    @staticmethod
    def _process_txt(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            texts = f.readlines()


from loguru import logger

class DataProcessor:
    @staticmethod
    def process_file(file_path: str, file_format: str) -> Tuple[List[str], List[Dict]]:
        """Process input file and return texts and metadata."""
        try:
            if file_format == 'csv':
                return DataProcessor._process_csv(file_path)
            elif file_format == 'json':
                return DataProcessor._process_json(file_path)
            elif file_format == 'txt':
                return DataProcessor._process_txt(file_path)
            else:
                raise ValueError(f"Unsupported file format: {file_format}")
        except Exception as e:
            logger.error(f"Error processing file: {e}")
            raise

    @staticmethod
    def _process_csv(file_path: str) -> Tuple[List[str], List[Dict]]:
        df = pd.read_csv(file_path)
        texts = df.iloc[:, 0].tolist()  # Assume first column contains text
        metadata = df.to_dict('records')
        return texts, metadata

    @staticmethod
    def _process_json(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            data = json.load(f)
        texts = [item.get('text', '') for item in data]
        metadata = data
        return texts, metadata

    @staticmethod
    def _process_txt(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            texts = f.readlines()
        metadata = [{'line_number': i} for i in range(len(texts))]
        return texts, metadata


            if file_format == 'csv':
                return DataProcessor._process_csv(file_path)
            elif file_format == 'json':
                return DataProcessor._process_json(file_path)
            elif file_format == 'txt':
                return DataProcessor._process_txt(file_path)
            else:
                raise ValueError(f"Unsupported file format: {file_format}")
        except Exception as e:
            logger.error(f"Error processing file: {e}")
            raise

    @staticmethod
    def _process_csv(file_path: str) -> Tuple[List[str], List[Dict]]:
        df = pd.read_csv(file_path)
        texts = df.iloc[:, 0].tolist()  # Assume first column contains text
        metadata = df.to_dict('records')
        return texts, metadata

    @staticmethod
    def _process_json(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            data = json.load(f)
        texts = [item.get('text', '') for item in data]
        metadata = data
        return texts, metadata

    @staticmethod
    def _process_txt(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            texts = f.readlines()
        metadata = [{'line_number': i} for i in range(len(texts))]
        return texts, metadata


        except Exception as e:
            logger.error(f"Error processing file: {e}")
            raise

    @staticmethod
    def _process_csv(file_path: str) -> Tuple[List[str], List[Dict]]:
        df = pd.read_csv(file_path)
        texts = df.iloc[:, 0].tolist()  # Assume first column contains text
        metadata = df.to_dict('records')
        return texts, metadata

    @staticmethod
    def _process_json(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            data = json.load(f)
        texts = [item.get('text', '') for item in data]
        metadata = data
        return texts, metadata

    @staticmethod
    def _process_txt(file_path: str) -> Tuple[List[str], List[Dict]]:
        with open(file_path, 'r') as f:
            texts = f.readlines()
        metadata = [{'line_number': i} for i in range(len(texts))]
        return texts, metadata


src.utils.logger

import json
from datetime import datetime
from typing import Any, Dict, Optional
import os
from pathlib import Path

class SystemLogger:
    def __init__(self, actions_log_path: str = "actions.log", thoughts_log_path: str = "thoughts.log"):
        """Initialize the system logger."""
        self.actions_log_path = actions_log_path
        self.thoughts_log_path = thoughts_log_path
        self._ensure_log_files_exist()

    def _ensure_log_files_exist(self):
        """Ensure log files exist and create them if they don't."""
        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)

    def _get_recent_entries(self, log_path: str, limit: int) -> list:
        """Get recent entries from a log file."""
        entries = []
        try:
            with open(log_path, "r", encoding="utf-8") as f:
                lines = f.readlines()
                for line in lines[-limit:]:
                    try:
                        entries.append(json.loads(line.strip()))
                    except json.JSONDecodeError:
                        continue
        except FileNotFoundError:
            pass
        return entries


import json
from datetime import datetime
from typing import Any, Dict, Optional
import os
from pathlib import Path

class SystemLogger:
    def __init__(self, actions_log_path: str = "actions.log", thoughts_log_path: str = "thoughts.log"):
        """Initialize the system logger."""
        self.actions_log_path = actions_log_path
        self.thoughts_log_path = thoughts_log_path
        self._ensure_log_files_exist()

    def _ensure_log_files_exist(self):
        """Ensure log files exist and create them if they don't."""
        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)


import json
from datetime import datetime
from typing import Any, Dict, Optional
import os
from pathlib import Path

class SystemLogger:
    def __init__(self, actions_log_path: str = "actions.log", thoughts_log_path: str = "thoughts.log"):
        """Initialize the system logger."""
        self.actions_log_path = actions_log_path
        self.thoughts_log_path = thoughts_log_path
        self._ensure_log_files_exist()

    def _ensure_log_files_exist(self):
        """Ensure log files exist and create them if they don't."""
        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""


import json
from datetime import datetime
from typing import Any, Dict, Optional
import os
from pathlib import Path

class SystemLogger:
    def __init__(self, actions_log_path: str = "actions.log", thoughts_log_path: str = "thoughts.log"):
        """Initialize the system logger."""
        self.actions_log_path = actions_log_path
        self.thoughts_log_path = thoughts_log_path
        self._ensure_log_files_exist()

    def _ensure_log_files_exist(self):
        """Ensure log files exist and create them if they don't."""
        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)


        self.actions_log_path = actions_log_path
        self.thoughts_log_path = thoughts_log_path
        self._ensure_log_files_exist()

    def _ensure_log_files_exist(self):
        """Ensure log files exist and create them if they don't."""
        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)

    def _get_recent_entries(self, log_path: str, limit: int) -> list:
        """Get recent entries from a log file."""
        entries = []
        try:
            with open(log_path, "r", encoding="utf-8") as f:


        for path in [self.actions_log_path, self.thoughts_log_path]:
            Path(path).touch(exist_ok=True)

    def _format_log_entry(self, entry_type: str, message: str, metadata: Optional[Dict[str, Any]] = None) -> str:
        """Format a log entry as JSON."""
        log_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)

    def _get_recent_entries(self, log_path: str, limit: int) -> list:
        """Get recent entries from a log file."""
        entries = []
        try:
            with open(log_path, "r", encoding="utf-8") as f:
                lines = f.readlines()
                for line in lines[-limit:]:
                    try:
                        entries.append(json.loads(line.strip()))
                    except json.JSONDecodeError:
                        continue


            "timestamp": datetime.utcnow().isoformat(),
            "type": entry_type,
            "message": message
        }
        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)

    def _get_recent_entries(self, log_path: str, limit: int) -> list:
        """Get recent entries from a log file."""
        entries = []
        try:
            with open(log_path, "r", encoding="utf-8") as f:
                lines = f.readlines()
                for line in lines[-limit:]:
                    try:
                        entries.append(json.loads(line.strip()))
                    except json.JSONDecodeError:
                        continue
        except FileNotFoundError:
            pass
        return entries


        if metadata:
            log_entry["metadata"] = metadata
        return json.dumps(log_entry)

    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)

    def _get_recent_entries(self, log_path: str, limit: int) -> list:
        """Get recent entries from a log file."""
        entries = []
        try:
            with open(log_path, "r", encoding="utf-8") as f:
                lines = f.readlines()
                for line in lines[-limit:]:
                    try:
                        entries.append(json.loads(line.strip()))
                    except json.JSONDecodeError:
                        continue
        except FileNotFoundError:
            pass
        return entries


    def log_action(self, action: str, metadata: Optional[Dict[str, Any]] = None):
        """Log an action with optional metadata."""
        log_entry = self._format_log_entry("action", action, metadata)
        with open(self.actions_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def log_thought(self, thought: str, metadata: Optional[Dict[str, Any]] = None):
        """Log a thought process with optional metadata."""
        log_entry = self._format_log_entry("thought", thought, metadata)
        with open(self.thoughts_log_path, "a", encoding="utf-8") as f:
            f.write(f"{log_entry}\n")

    def get_recent_actions(self, limit: int = 10) -> list:
        """Get the most recent actions from the log."""
        return self._get_recent_entries(self.actions_log_path, limit)

    def get_recent_thoughts(self, limit: int = 10) -> list:
        """Get the most recent thoughts from the log."""
        return self._get_recent_entries(self.thoughts_log_path, limit)

    def _get_recent_entries(self, log_path: str, limit: int) -> list:
        """Get recent entries from a log file."""
        entries = []
        try:
            with open(log_path, "r", encoding="utf-8") as f:
                lines = f.readlines()
                for line in lines[-limit:]:
                    try:
                        entries.append(json.loads(line.strip()))
                    except json.JSONDecodeError:
                        continue
        except FileNotFoundError:
            pass
        return entries


src.api.main

from fastapi import FastAPI, UploadFile, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
from typing import List, Optional, Dict
import yaml
from loguru import logger
import sys
import os
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database.vector_store import VectorStore
from chatgpt.knowledge_processor import KnowledgeProcessor

app = FastAPI(title="RAG Vector Database API")

# CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Load configuration
with open("config.yaml", 'r') as f:
    config = yaml.safe_load(f)

# Initialize vector store and knowledge processor
vector_store = VectorStore()
knowledge_processor = KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

class QueryRequest(BaseModel):
    query: str
    k: int = Field(default=5, gt=0, description="Number of results to return")

class AddTextsRequest(BaseModel):
    texts: List[str]
    metadata: Optional[List[Dict]] = None

class GenerateKnowledgeRequest(BaseModel):
    user_input: str

@app.post("/generate-knowledge")
async def generate_knowledge(request: GenerateKnowledgeRequest):
    """Generate agent knowledge from user input and store it in the vector database."""
    try:
        # Process user input to generate knowledge
        knowledge = knowledge_processor.process_user_input(request.user_input)

        # Store the generated knowledge in vector database
        ids = vector_store.add_texts(
            texts=knowledge["text_chunks"],
            metadata=knowledge["metadata"]
        )

        return {
            "message": "Successfully generated and stored agent knowledge",
            "structured_knowledge": knowledge["structured_knowledge"],
            "vector_ids": ids
        }
    except Exception as e:
        logger.error(f"Knowledge generation error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/search")
async def search(request: QueryRequest):
    try:
        results = vector_store.search(request.query, k=request.k)
        return {"results": results}
    except Exception as e:
        logger.error(f"Search error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/add-texts")
async def add_texts(request: AddTextsRequest):
    try:
        ids = vector_store.add_texts(request.texts, request.metadata)
        return {"ids": ids}
    except Exception as e:
        logger.error(f"Add texts error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")
async def health_check():
    return {"status": "healthy"}

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(
        "main:app",
        host=config['api']['host'],
        port=config['api']['port'],
        reload=True
    )


QueryRequest

app = FastAPI(title="RAG Vector Database API")

# CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Load configuration
with open("config.yaml", 'r') as f:
    config = yaml.safe_load(f)

# Initialize vector store and knowledge processor
vector_store = VectorStore()
knowledge_processor = KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

class QueryRequest(BaseModel):
    query: str
    k: int = Field(default=5, gt=0, description="Number of results to return")

class AddTextsRequest(BaseModel):
    texts: List[str]
    metadata: Optional[List[Dict]] = None

class GenerateKnowledgeRequest(BaseModel):
    user_input: str

@app.post("/generate-knowledge")
async def generate_knowledge(request: GenerateKnowledgeRequest):
    """Generate agent knowledge from user input and store it in the vector database."""
    try:
        # Process user input to generate knowledge
        knowledge = knowledge_processor.process_user_input(request.user_input)

        # Store the generated knowledge in vector database
        ids = vector_store.add_texts(
            texts=knowledge["text_chunks"],
            metadata=knowledge["metadata"]
        )


AddTextsRequest

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Load configuration
with open("config.yaml", 'r') as f:
    config = yaml.safe_load(f)

# Initialize vector store and knowledge processor
vector_store = VectorStore()
knowledge_processor = KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

class QueryRequest(BaseModel):
    query: str
    k: int = Field(default=5, gt=0, description="Number of results to return")

class AddTextsRequest(BaseModel):
    texts: List[str]
    metadata: Optional[List[Dict]] = None

class GenerateKnowledgeRequest(BaseModel):
    user_input: str

@app.post("/generate-knowledge")
async def generate_knowledge(request: GenerateKnowledgeRequest):
    """Generate agent knowledge from user input and store it in the vector database."""
    try:
        # Process user input to generate knowledge
        knowledge = knowledge_processor.process_user_input(request.user_input)

        # Store the generated knowledge in vector database
        ids = vector_store.add_texts(
            texts=knowledge["text_chunks"],
            metadata=knowledge["metadata"]
        )

        return {
            "message": "Successfully generated and stored agent knowledge",
            "structured_knowledge": knowledge["structured_knowledge"],


GenerateKnowledgeRequest

    allow_methods=["*"],
    allow_headers=["*"],
)

# Load configuration
with open("config.yaml", 'r') as f:
    config = yaml.safe_load(f)

# Initialize vector store and knowledge processor
vector_store = VectorStore()
knowledge_processor = KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

class QueryRequest(BaseModel):
    query: str
    k: int = Field(default=5, gt=0, description="Number of results to return")

class AddTextsRequest(BaseModel):
    texts: List[str]
    metadata: Optional[List[Dict]] = None

class GenerateKnowledgeRequest(BaseModel):
    user_input: str

@app.post("/generate-knowledge")
async def generate_knowledge(request: GenerateKnowledgeRequest):
    """Generate agent knowledge from user input and store it in the vector database."""
    try:
        # Process user input to generate knowledge
        knowledge = knowledge_processor.process_user_input(request.user_input)

        # Store the generated knowledge in vector database
        ids = vector_store.add_texts(
            texts=knowledge["text_chunks"],
            metadata=knowledge["metadata"]
        )

        return {
            "message": "Successfully generated and stored agent knowledge",
            "structured_knowledge": knowledge["structured_knowledge"],
            "vector_ids": ids
        }
    except Exception as e:


# Load configuration
with open("config.yaml", 'r') as f:
    config = yaml.safe_load(f)

# Initialize vector store and knowledge processor
vector_store = VectorStore()
knowledge_processor = KnowledgeProcessor(api_key=os.getenv('OPENAI_API_KEY'))

class QueryRequest(BaseModel):
    query: str
    k: int = Field(default=5, gt=0, description="Number of results to return")

class AddTextsRequest(BaseModel):
    texts: List[str]
    metadata: Optional[List[Dict]] = None

class GenerateKnowledgeRequest(BaseModel):
    user_input: str

@app.post("/generate-knowledge")
async def generate_knowledge(request: GenerateKnowledgeRequest):
    """Generate agent knowledge from user input and store it in the vector database."""
    try:
        # Process user input to generate knowledge
        knowledge = knowledge_processor.process_user_input(request.user_input)

        # Store the generated knowledge in vector database
        ids = vector_store.add_texts(
            texts=knowledge["text_chunks"],
            metadata=knowledge["metadata"]
        )

        return {
            "message": "Successfully generated and stored agent knowledge",
            "structured_knowledge": knowledge["structured_knowledge"],
            "vector_ids": ids
        }
    except Exception as e:
        logger.error(f"Knowledge generation error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/search")
async def search(request: QueryRequest):
    try:
        results = vector_store.search(request.query, k=request.k)
        return {"results": results}
    except Exception as e:
        logger.error(f"Search error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/add-texts")
async def add_texts(request: AddTextsRequest):
    try:
        ids = vector_store.add_texts(request.texts, request.metadata)
        return {"ids": ids}
    except Exception as e:
        logger.error(f"Add texts error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")