initial commit

.gitignore

@@ -0,0 +1,6 @@
+.env
+__pycache__
+littlejs
+.DS_Store
+venv
+hf_token

README.md

@@ -1,13 +1,37 @@
 ---
 title: AI Project
-emoji: 👀
+emoji: 👾
+colorFrom: blue
 colorFrom: yellow
 colorTo: pink
 sdk: gradio
 sdk_version: 5.22.0
 app_file: app.py
 pinned: false
-short_description: Chat bot with RAG fo LittleJS game engine
+short_description: Chat bot with RAG for LittleJS game engine
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# TowardsAI Course final project
+
+By Eoin McGrath eoin.mcg@gmail.com
+
+## Title
+Game dev tutor focusing on LittleJS framework
+
+## Overview
+Data gathered from offical docs and github repo source.
+Processed and generated embeddings stored in Chroma DB.
+Evaluation scripts and data provided.
+Reranker used to improve generated answers.
+
+## Optional Extras
+1.  Implement streaming responses.
+2.  The app is designed for a specific goal/domain that is not a tutor about AI: designed for a specific javascript based game engine
+3.  You have shown evidence of collecting at least two data sources beyond those provided in our course: 
+  fetch_docs.py = output stored in ./data/littlejs_docs.csv
+  fetch_repo.py = output stored in ./data/littlejs_repo.csv
+4  There’s code for RAG evaluation in the folder, and the README contains the evaluation results:
+  eval_generate_dataset.py = generates synthetic question context  pairs 
+  eval_process_dataset.py  = evaluation results based on above questions
+  eval_results.txt         = sample saved results
+5. Use a reranker in your RAG pipeline. It can be a fine-tuned version (your choice): uses LLMRerank postprocessor

app.py

@@ -0,0 +1,157 @@
+import logging
+import os
+
+import gradio as gr
+
+from llama_index.core.retrievers import VectorIndexRetriever
+from llama_index.core.llms import MessageRole
+from llama_index.core.memory import ChatSummaryMemoryBuffer
+from llama_index.core.tools import RetrieverTool, ToolMetadata
+from llama_index.agent.openai import OpenAIAgent
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.llms.openai import OpenAI
+from llama_index.core import Settings
+from llama_index.core.postprocessor import LLMRerank
+
+from utils import create_db, load_db, load_asset
+from config import CHROMA_PATH, PLACEHOLDER, TITLE, PROMPT_SYSTEM_MESSAGE, TEXT_QA_TEMPLATE
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+
+API_KEY=""
+token_count = 0
+
+def create_knowledge_base_if_not_exists():
+    if not os.path.exists(CHROMA_PATH) or not os.listdir(CHROMA_PATH):
+        print("⚠️ ChromaDB not found. Creating DB...")
+        create_db()
+
+def get_tools():
+    index = load_db()
+    vector_retriever = VectorIndexRetriever(
+        index=index,
+        similarity_top_k=15,
+        embed_model=Settings.embed_model,
+        use_async=True,
+    )
+
+    # Add LLMRerank for better retrieval
+    reranker = LLMRerank(
+        choice_batch_size=5,
+        top_n=3,
+    )
+
+    def retrieve_with_rerank(query):
+        retrieved_docs = vector_retriever.retrieve(query)
+        reranked_docs = reranker.postprocess(retrieved_docs)
+        return reranked_docs
+
+    tools = [
+        RetrieverTool(
+            # retriever=vector_retriever,
+            retriever=retrieve_with_rerank,
+            metadata=ToolMetadata(
+                name="LitleJS_related_resources",
+                description="Useful for info related to the LittleJS game development library. It gathers the info from local data.",
+            ),
+        )
+    ]
+    return tools
+
+def set_api_key(key):
+    API_KEY=key
+    Settings.llm = OpenAI(temperature=0, model="gpt-4o-mini", api_key=API_KEY)
+    Settings.embed_model = OpenAIEmbedding(model="text-embedding-3-small")
+
+
+def generate_completion(query, history, memory, api_key):
+    logging.info(f"User query: {query}")
+
+    if not API_KEY:
+        set_api_key(api_key)
+
+    # Manage memory
+    chat_list = memory.get()
+    if len(chat_list) != 0:
+        user_index = [i for i, msg in enumerate(chat_list) if msg.role == MessageRole.USER]
+        if len(user_index) > len(history):
+            user_index_to_remove = user_index[len(history)]
+            chat_list = chat_list[:user_index_to_remove]
+            memory.set(chat_list)
+    logging.info(f"chat_history: {len(memory.get())} {memory.get()}")
+    logging.info(f"gradio_history: {len(history)} {history}")
+
+    # Create agent
+    tools = get_tools()
+
+    agent = OpenAIAgent.from_tools(
+        llm=Settings.llm,
+        memory=memory,
+        tools=tools,
+        system_prompt=PROMPT_SYSTEM_MESSAGE,
+    )
+
+    # Generate answer
+    completion = agent.stream_chat(query)
+    answer_str = ""
+    for token in completion.response_gen:
+        answer_str += token
+        global token_count
+        token_count += 1  # Update token count
+        yield answer_str
+
+
+
+def launch_ui():
+
+    js=load_asset("./assets/chat.js")
+
+    with gr.Blocks(
+        title=TITLE,
+        fill_height=True,
+        analytics_enabled=True,
+        css=load_asset("./assets/style.css"),
+        js=load_asset("./assets/chat.js"),
+    ) as demo:
+
+        api_key_input = gr.Textbox(
+            label="Enter your OpenAI API Key",
+            type="password",
+            placeholder="sk-...",
+            elem_classes="api_key_input"
+        )
+
+        memory_state = gr.State(
+            lambda: ChatSummaryMemoryBuffer.from_defaults(
+                token_limit=120000,
+            )
+        )
+        chatbot = gr.Chatbot(
+            scale=1,
+            placeholder=PLACEHOLDER,
+            type='messages',
+            show_label=False,
+            show_copy_button=True,
+            elem_classes="chatbox",
+        )
+
+        gr.ChatInterface(
+            fn=generate_completion,
+            chatbot=chatbot,
+            type='messages',
+            additional_inputs=[memory_state, api_key_input],
+        )
+
+        token_counter = gr.Button("Tokens Used: 0", elem_classes="token_counter")
+
+        demo.queue(default_concurrency_limit=64)
+        demo.launch(debug=True, favicon_path="./assets/favicon.png", share=False) # Set share=True to share the app online
+
+
+if __name__ == "__main__":
+    create_knowledge_base_if_not_exists()
+    launch_ui()

assets/chat.js

@@ -0,0 +1,60 @@
+function createGradioAnimation() {
+    var container = document.createElement('div');
+    container.id = 'gradio-animation';
+    container.style.fontSize = '2em';
+    container.style.fontWeight = 'bold';
+    container.style.textAlign = 'center';
+    container.style.marginBottom = '20px';
+
+    var text = 'LittleJS Game Tutor 👾';
+    for (var i = 0; i < text.length; i++) {
+        (function(i){
+            setTimeout(function(){
+                var letter = document.createElement('span');
+                letter.style.opacity = '0';
+                letter.style.transition = 'opacity 0.5s';
+                letter.innerText = text[i];
+
+                container.appendChild(letter);
+
+                setTimeout(function() {
+                    letter.style.opacity = '1';
+                }, 20);
+            }, i * 50);
+        })(i);
+    }
+
+    var gradioContainer = document.querySelector('.gradio-container');
+    gradioContainer.insertBefore(container, gradioContainer.firstChild);
+
+
+      const showChat = () => {
+        header.style.display = 'none';
+        chatbox.style.display = 'block';
+        chatinput.style.display = 'block'
+        token_counter.style.display = 'none';
+      }
+
+      const header = container;
+      const input = document.querySelector('.api_key_input');
+      input.focus();
+      const chatbox = document.querySelector('.chatbox');
+      const chatinput = document.querySelectorAll('.form')[1];
+      const token_counter = document.querySelector('.token_counter');
+
+    
+      chatbox.style.display = 'none';
+      chatinput.style.display = 'none';
+      token_counter.style.display = 'none';
+
+      input.addEventListener(('blur'), (e) => {
+        showChat();
+      })
+      input.addEventListener(('paste'), (e) => {
+        showChat();
+      })
+
+    return 'Animation created';
+}
+
+

assets/style.css

@@ -0,0 +1,103 @@
+footer { display: none !important; }
+/* Custom CSS for LittleJS Chatbot */
+
+.placeholder { text-align: center; }
+.hide { display: none; }
+
+/* Main container styling */
+body {
+    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+}
+
+/* API Key input styling */
+.api-key-container {
+    max-width: 600px;
+    margin: 50px auto;
+    padding: 20px;
+    border-radius: 10px;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
+    background-color: #f8f9fa;
+}
+
+/* Chat container */
+.chat-container {
+    border-radius: 10px;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.05);
+    background-color: #ffffff;
+}
+
+/* Message bubbles */
+.message-bubble {
+    padding: 12px 16px;
+    border-radius: 16px;
+    max-width: 85%;
+    margin-bottom: 10px;
+}
+
+.user-message {
+    background-color: #e3f2fd;
+    align-self: flex-end;
+}
+
+.bot-message {
+    background-color: #f1f3f4;
+    align-self: flex-start;
+}
+
+/* Token display */
+.token-display {
+    background-color: #f8f9fa;
+    border-radius: 8px;
+    padding: 10px;
+    font-size: 0.9em;
+    border-left: 4px solid #4285f4;
+}
+
+/* Buttons */
+.primary-button {
+    background-color: #4285f4;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    padding: 8px 16px;
+    cursor: pointer;
+    transition: background-color 0.2s;
+}
+
+.primary-button:hover {
+    background-color: #3367d6;
+}
+
+.secondary-button {
+    background-color: #f1f3f4;
+    color: #202124;
+    border: 1px solid #dadce0;
+    border-radius: 4px;
+    padding: 8px 16px;
+    cursor: pointer;
+    transition: background-color 0.2s;
+}
+
+.secondary-button:hover {
+    background-color: #e8eaed;
+}
+
+/* Code blocks */
+pre {
+    background-color: #f5f7f9;
+    border-radius: 6px;
+    padding: 12px;
+    overflow-x: auto;
+}
+
+code {
+    font-family: 'Courier New', Courier, monospace;
+    font-size: 0.9em;
+}
+
+/* Error message */
+.error-message {
+    color: #d32f2f;
+    font-size: 0.9em;
+    margin-top: 8px;
+}

config.py

@@ -0,0 +1,29 @@
+TITLE="LittleJS Game Tutor 👾"
+PLACEHOLDER="<h1>LittleJS Game Tutor 👾</h1><br> A Question-Answering Bot for anything LittleJS related</strong><br>"
+CHROMA_PATH="littlejs"
+CHROMA_COLLECTION="littlejs"
+CHUNK_SIZE=512
+CHUNK_OVERLAP=128
+FILES=["./data/littlejs_docs.csv", "./data/littlejs_repo.csv"]
+# FILES=["./data/littlejs_docs.csv"]
+
+PROMPT_SYSTEM_MESSAGE = """You are a programming teacher, answering questions from students of a course on game development using the LittleJS library.
+Topics covered include installing, set up, creating games with LittleJS, debugging techniques etc. Questions should be understood in this context. Your answers are aimed to teach 
+students, so they should be complete, clear, and easy to understand. Use the available tools to gather insights pertinent to game development with LittleJS.
+To find relevant information for answering student questions, always use the "LitleJS_related_resources" tool.
+
+Only some information returned by the tool might be relevant to the question, so ignore the irrelevant part and answer the question with what you have. Your responses are exclusively based on the output provided 
+by the tools. Refrain from incorporating information not directly obtained from the tool's responses.
+If a user requests further elaboration on a specific aspect of a previously discussed topic, you should reformulate your input to the tool to capture this new angle or more profound layer of inquiry. Provide 
+comprehensive answers, ideally structured in multiple paragraphs, drawing from the tool's variety of relevant details. Provide code samples where possible. The depth and breadth of your responses should align with the scope and specificity of the information retrieved. 
+Should the tool response lack information on the queried topic, politely inform the user that the question transcends the bounds of your current knowledge base, citing the absence of relevant content in the tool's documentation. 
+At the end of your answers, always invite the students to ask deeper questions about the topic if they have any.
+Do not refer to the documentation directly, but use the information provided within it to answer questions. If code is provided in the information, share it with the students. It's important to provide complete code blocks so 
+they can execute the code when they copy and paste them. Make sure to format your answers in Markdown format, including code blocks and snippets.
+"""
+
+TEXT_QA_TEMPLATE = """
+You must answer only related to LittleJS, game development and related concepts queries.
+Always leverage the retrieved documents to answer the questions, don't answer them on your own.
+If the query is not relevant to LittleJS, say that you don't know the answer.
+"""

data/extra_resources.txt

@@ -0,0 +1,2 @@
+https://raw.githubusercontent.com/KilledByAPixel/LittleJS/refs/heads/main/FAQ.mhttps://raw.githubusercontent.com/KilledByAPixel/LittleJS/refs/heads/main/FAQ.md
+https://raw.githubusercontent.com/KilledByAPixel/LittleJS/refs/heads/main/examples/breakoutTutorial/README.md

data/littlejsdocs.txt

@@ -0,0 +1,28 @@
+https://killedbyapixel.github.io/LittleJS/docs/
+https://killedbyapixel.github.io/LittleJS/docs/Color.html
+https://killedbyapixel.github.io/LittleJS/docs/EngineObject.html
+https://killedbyapixel.github.io/LittleJS/docs/FontImage.html
+https://killedbyapixel.github.io/LittleJS/docs/Medal.html
+https://killedbyapixel.github.io/LittleJS/docs/Music.html
+https://killedbyapixel.github.io/LittleJS/docs/Particle.html
+https://killedbyapixel.github.io/LittleJS/docs/ParticleEmitter.html
+https://killedbyapixel.github.io/LittleJS/docs/RandomGenerator.html
+https://killedbyapixel.github.io/LittleJS/docs/Sound.html
+https://killedbyapixel.github.io/LittleJS/docs/SoundWave.html
+https://killedbyapixel.github.io/LittleJS/docs/TextureInfo.html
+https://killedbyapixel.github.io/LittleJS/docs/TileInfo.html
+https://killedbyapixel.github.io/LittleJS/docs/TileLayer.html
+https://killedbyapixel.github.io/LittleJS/docs/TileLayerData.html
+https://killedbyapixel.github.io/LittleJS/docs/Timer.html
+https://killedbyapixel.github.io/LittleJS/docs/Vector2.html
+https://killedbyapixel.github.io/LittleJS/docs/Audio.html
+https://killedbyapixel.github.io/LittleJS/docs/Debug.html
+https://killedbyapixel.github.io/LittleJS/docs/Draw.html
+https://killedbyapixel.github.io/LittleJS/docs/Engine.html
+https://killedbyapixel.github.io/LittleJS/docs/Input.html
+https://killedbyapixel.github.io/LittleJS/docs/Medals.html
+https://killedbyapixel.github.io/LittleJS/docs/Random.html
+https://killedbyapixel.github.io/LittleJS/docs/Settings.html
+https://killedbyapixel.github.io/LittleJS/docs/TileCollision.html
+https://killedbyapixel.github.io/LittleJS/docs/Utilities.html
+https://killedbyapixel.github.io/LittleJS/docs/WebGL.html

eval_generate_dataset.py

@@ -0,0 +1,117 @@
+import os
+import json
+from dotenv import load_dotenv
+
+from llama_index.core import Settings
+from llama_index.embeddings.openai import OpenAIEmbedding
+
+from llama_index.llms.openai import OpenAI
+from llama_index.llms.gemini import Gemini
+
+from llama_index.core.evaluation import generate_question_context_pairs
+from llama_index.core.evaluation import RetrieverEvaluator
+from llama_index.core.query_engine import RetrieverQueryEngine
+
+from utils import create_db
+from config import CHROMA_PATH, CHROMA_COLLECTION
+
+load_dotenv()
+
+Settings.llm = OpenAI(temperature=0, model="gpt-4o-mini", max_tokens=512)
+Settings.embed_model = OpenAIEmbedding(
+    model="text-embedding-3-small"
+)
+
+nodes = create_db(return_nodes=True)
+
+# Free Tier-Gemini API key
+from llama_index.core.llms.utils import LLM
+from llama_index.core.schema import MetadataMode, TextNode
+from tqdm import tqdm
+import json
+import re
+import uuid
+import warnings
+import time
+from typing import Dict, List, Tuple
+from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
+
+DEFAULT_QA_GENERATE_PROMPT_TMPL = """\
+Context information is below.
+
+---------------------
+{context_str}
+---------------------
+
+Given the context information and not prior knowledge.
+generate only questions based on the below query.
+
+You are a Teacher/ Professor. Your task is to setup \
+{num_questions_per_chunk} questions for an upcoming \
+quiz/examination. The questions should be diverse in nature \
+across the document. Restrict the questions to the \
+context information provided."
+"""
+
+def generate_question_context_pairs(
+    nodes: List[TextNode],
+    llm: LLM,
+    qa_generate_prompt_tmpl: str = DEFAULT_QA_GENERATE_PROMPT_TMPL,
+    num_questions_per_chunk: int = 2,
+    request_delay: float = 2.0
+) -> EmbeddingQAFinetuneDataset:
+    """Generate examples given a set of nodes with delays between requests."""
+    node_dict = {
+        node.node_id: node.get_content(metadata_mode=MetadataMode.NONE)
+        for node in nodes
+    }
+
+    queries = {}
+    relevant_docs = {}
+
+    for node_id, text in tqdm(node_dict.items()):
+        query = qa_generate_prompt_tmpl.format(
+            context_str=text, num_questions_per_chunk=num_questions_per_chunk
+        )
+        response = llm.complete(query)
+
+        result = str(response).strip().split("\n")
+        questions = [
+            re.sub(r"^\d+[\).\s]", "", question).strip() for question in result
+        ]
+        questions = [question for question in questions if len(question) > 0][
+            :num_questions_per_chunk
+        ]
+
+        num_questions_generated = len(questions)
+        if num_questions_generated < num_questions_per_chunk:
+            warnings.warn(
+                f"Fewer questions generated ({num_questions_generated}) "
+                f"than requested ({num_questions_per_chunk})."
+            )
+
+        for question in questions:
+            question_id = str(uuid.uuid4())
+            queries[question_id] = question
+            relevant_docs[question_id] = [node_id]
+
+        time.sleep(request_delay)
+
+    return EmbeddingQAFinetuneDataset(
+        queries=queries, corpus=node_dict, relevant_docs=relevant_docs
+    )
+
+#from llama_index.core.evaluation import generate_question_context_pairs
+from llama_index.llms.gemini import Gemini
+
+llm = Gemini(model="models/gemini-1.5-flash", temperature=1, max_tokens=512)
+
+rag_eval_dataset = generate_question_context_pairs(
+    nodes[:25],
+    llm=llm,
+    num_questions_per_chunk=1,
+    request_delay=4
+)
+
+# Save the dataset as a json file for later use
+rag_eval_dataset.save_json("./rag_eval_dataset.json")

eval_process_dataset.py

@@ -0,0 +1,76 @@
+import pandas as pd
+from dotenv import load_dotenv
+from llama_index.core import Settings
+from llama_index.core.callbacks import CallbackManager, LlamaDebugHandler
+
+from llama_index.core.evaluation import RetrieverEvaluator
+from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
+from llama_index.core.evaluation import RelevancyEvaluator, FaithfulnessEvaluator, BatchEvalRunner
+from llama_index.llms.openai import OpenAI
+from utils import load_db  # Assuming utils.py is in the same directory
+
+from llama_index.llms.openai import OpenAI
+from llama_index.embeddings.openai import OpenAIEmbedding
+
+import asyncio
+import nest_asyncio
+
+# Apply nest_asyncio to allow nested event loops
+nest_asyncio.apply()
+
+Settings.embed_model = OpenAIEmbedding(
+    model="text-embedding-ada-002",
+    timeout=60,  # Increase timeout to 60 seconds
+    max_retries=5,  # Increase max retries
+    retry_delay=2  # Wait 2 seconds between retries
+)
+# Create a debug handler to see more info about API calls
+# debug_handler = LlamaDebugHandler(print_trace_on_end=True)
+# callback_manager = CallbackManager([debug_handler])
+
+# Configure global settings
+# Settings.callback_manager = callback_manager
+Settings.retry_policy = {
+    "max_retries": 5,
+    "retry_delay": 2.0,
+    "exponential_backoff": True
+}
+async def run_evaluation():
+    load_dotenv()
+    index = load_db()
+    rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json("./rag_eval_dataset.json")
+    # Define an LLM as a judge
+    llm_gpt4o = OpenAI(temperature=0, model="gpt-4o")
+    llm_gpt4o_mini = OpenAI(temperature=0, model="gpt-4o-mini")
+    # Initiate the faithfulnes and relevancy evaluator objects
+    faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_gpt4o)
+    relevancy_evaluator = RelevancyEvaluator(llm=llm_gpt4o)
+    # Extract the questions from the dataset
+    queries = list(rag_eval_dataset.queries.values())
+    # Limit to first 20 question to save time (!!remove this line in production!!)
+    batch_eval_queries = queries[:5]
+    # The batch evaluator runs the evaluation in batches
+    runner = BatchEvalRunner(
+        {"faithfulness": faithfulness_evaluator, "relevancy": relevancy_evaluator},
+        workers=8,
+    )
+    # Define a for-loop to try different `similarity_top_k` values
+    for i in [2,4,6]:
+        # Set query engine with different number of returned chunks
+        query_engine = index.as_query_engine(similarity_top_k=i, llm=llm_gpt4o_mini)
+        # Run the evaluation
+        eval_results = await runner.aevaluate_queries(query_engine, queries=batch_eval_queries)
+        # Printing the results
+        faithfulness_score = sum(
+            result.passing for result in eval_results["faithfulness"]
+        ) / len(eval_results["faithfulness"])
+        print(f"top_{i} faithfulness_score: {faithfulness_score}")
+        relevancy_score = sum(result.passing for result in eval_results["relevancy"]) / len(
+            eval_results["relevancy"]
+        )
+        print(f"top_{i} relevancy_score: {relevancy_score}")
+        print("="*15)
+
+# Run the async function
+if __name__ == "__main__":
+    asyncio.run(run_evaluation())

eval_results.txt

@@ -0,0 +1,9 @@
+top_2 faithfulness_score: 0.4
+top_2 relevancy_score: 0.6
+===============
+top_4 faithfulness_score: 0.5
+top_4 relevancy_score: 0.9
+===============
+top_6 faithfulness_score: 0.4
+top_6 relevancy_score: 0.75
+===============

fetch_docs.py

@@ -0,0 +1,34 @@
+import csv
+import requests
+from bs4 import BeautifulSoup
+
+with open("./data/littlejsdocs.txt") as file:
+    urls = file.readlines()
+    # Remove trailing newline characters
+    urls = [url.rstrip('\n') for url in urls if not url.startswith('#')]
+
+def parse_webpage(url: str):
+    html = requests.get(url).text
+    soup = BeautifulSoup(html, "html.parser")
+    title = soup.find("title").get_text()
+    text = soup.find("div", class_="main-wrapper")
+    text.find('footer').extract()
+    return [
+        title,
+        text.get_text(),
+        url
+    ]
+
+docs = []
+for url in urls:
+    docs.append(parse_webpage(url))
+
+with open('./data/littledocs.csv', 'w', newline='') as file:
+    writer = csv.writer(file)
+    field = ["title", "text", "url"]
+
+    writer.writerow(field)
+    for line in docs:
+        writer.writerow([line[0], line[1], line[2]])
+
+print('DONE')

fetch_repo.py

@@ -0,0 +1,106 @@
+import argparse
+import os
+import requests
+import csv
+from urllib.parse import urljoin
+
+from llama_index.core import Document
+
+FILE="repo_files.csv"
+
+def get_github_repo_files(repo_url):
+    """
+    Fetches all JavaScript files in the 'src' directory of a GitHub repository.
+
+    Args:
+        repo_url (str): The URL of the GitHub repository.
+
+    Returns:
+        list: A list of dictionaries, each containing file information (filename, source, url).
+    """
+    try:
+        repo_name = repo_url.split("github.com/")[1].rstrip("/")
+        api_url = f"https://api.github.com/repos/{repo_name}/contents/src"
+        response = requests.get(api_url)
+        response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
+        files = response.json()
+        js_files = []
+
+        for file in files:
+            if file["type"] == "file" and file["name"].endswith(".js"):
+                file_info = {
+                    "filename": file["name"],
+                    "url": file["download_url"],
+                }
+                file_content_response = requests.get(file["download_url"])
+                file_content_response.raise_for_status()
+                file_info["source"] = file_content_response.text
+                js_files.append(file_info)
+
+        return js_files
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching repository files: {e}")
+        return []
+    except IndexError:
+        print("Invalid GitHub repository URL format.")
+        return []
+    except KeyError:
+        print("src directory not found in the repository")
+        return []
+    except Exception as e:
+        print(f"An unexpected error occured: {e}")
+        return []
+
+def write_to_csv(data, output_file=FILE):
+    """
+    Writes file information to a CSV file.
+
+    Args:
+        data (list): A list of dictionaries, each containing file information.
+        output_file (str): The name of the output CSV file.
+    """
+    try:
+        with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
+            fieldnames = ["filename", "source", "url"]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+            writer.writeheader()
+            for file_info in data:
+                writer.writerow(file_info)
+        print(f"File information written to {output_file}")
+    except Exception as e:
+        print(f"Error writing to CSV: {e}")
+
+def create_csv():
+    """
+    Main function to parse arguments and process the repository.
+    """
+    parser = argparse.ArgumentParser(description="Fetch JavaScript files from a GitHub repository's src directory.")
+    parser.add_argument("repo_url", help="The GitHub repository URL.")
+
+    repo_url = "https://github.com/KilledByAPixel/LittleJS"
+
+    files = get_github_repo_files(repo_url)
+    if files:
+        write_to_csv(files)
+
+
+print('READY?')
+
+rows = []
+# Load the file as a JSON
+with open(FILE, mode="r", encoding="utf-8") as file:
+    csv_reader = csv.reader(file)
+
+    for idx, row in enumerate(csv_reader):
+        if idx == 0: continue; # Skip header row
+        rows.append(row)
+
+# Convert the chunks to Document objects so the LlamaIndex framework can process them.
+documents = [Document(text=row[1], metadata={"title": row[0], "url": row[2]}) for row in rows]
+print(len(documents))
+print(documents[0].metadata)
+print(documents[0])
+
+

optional.txt

@@ -0,0 +1,41 @@
+1. [X] Implement streaming responses.
+
+?? Uses dynamic few-shot prompting, where the best examples are selected according to the user query.
+
+2. [] There’s code for RAG evaluation in the folder, and the README contains the evaluation results. 
+The folder must also contain the evaluation dataset and the evaluation scripts.
+https://academy.towardsai.net/courses/take/beginner-to-advanced-llm-dev/multimedia/59791119-evaluating-your-rag-pipeline
+
+3. [X] The app is designed for a specific goal/domain that is not a tutor about AI. For example, it could be about finance, healthcare, etc.
+
+Use live search results. In this case, the user must also input the API keys for the relevant APIs (e.g., Perplexity, Bing search, etc.).
+
+4. [X] You have shown evidence of collecting at least two data sources beyond those provided in our course. 
+
+Your data collection and curation process leverages structured JSON outputs, which are used for advanced RAG functionalities in your app.
+
+Your data collection and curation process leverages images and/or PDFs. The parsed data should be useful for some use cases in your app.
+
+The app can generate images on the fly. They must be pertinent to the scope of your app.
+
+Use a reranker in your RAG pipeline. It can be a fine-tuned version (your choice).
+
+Use hybrid search in your RAG pipeline.
+
+5. [ ] Use metadata filtering.
+
+Use a fine-tuned LLM in your app.
+
+Use a fine-tuned embedding model in your app. 
+
+Your RAG pipeline includes query routing.
+https://academy.towardsai.net/courses/take/beginner-to-advanced-llm-dev/multimedia/59791736-adding-question-validation-and-routing
+
+Your query pipeline includes function calling.
+
+Your app manages speech inputs.
+
+Your app manages speech outputs.
+
+Your app appropriately leverages context caching (also known as prompt caching). Explain in the README how you designed your prompts to be adequate for context caching.
+https://academy.towardsai.net/courses/take/beginner-to-advanced-llm-dev/multimedia/59791739-long-context-llms-context-caching-vs-rag

requirements.txt

@@ -0,0 +1,12 @@
+beautifulsoup4==4.13.3
+chromadb==0.6.3
+gradio==5.22.0
+huggingface_hub==0.29.3
+llama_index==0.12.25
+nest_asyncio==1.6.0
+openai==1.68.0
+pandas==2.2.3
+python-dotenv==1.0.1
+Requests==2.32.3
+tiktoken==0.9.0
+tqdm==4.67.1

test_query.py

@@ -0,0 +1,37 @@
+import os
+from dotenv import load_dotenv
+
+from llama_index.core import Settings
+from llama_index.embeddings.openai import OpenAIEmbedding
+
+from llama_index.llms.openai import OpenAI
+from llama_index.llms.gemini import Gemini
+
+from utils import load_db, create_db
+from config import CHROMA_PATH, CHROMA_COLLECTION, FILES, CHUNK_SIZE, CHUNK_OVERLAP
+
+load_dotenv()
+
+Settings.llm = OpenAI(temperature=0, model="gpt-4o-mini")
+Settings.embed_model = OpenAIEmbedding(
+    model="text-embedding-3-small"
+)
+
+if not os.path.exists(CHROMA_PATH) or not os.listdir(CHROMA_PATH):
+    print("⚠️ ChromaDB not found. Creating DB...")
+    index = create_db()
+else:
+    print("✅ ChromaDB found. Loading DB...", CHROMA_PATH, CHROMA_COLLECTION)
+    index = load_db()
+
+
+q="What is LittleJS?"
+q="How to install?"
+q="Who is the author?"
+# q="How to boil an egg?"
+# q="What params does the EngineObject require?"
+# q="What is vec2?"
+query_engine = index.as_query_engine(llm=Settings.llm, similarity_top_k=5)
+res = query_engine.query(q)
+
+print(res.response)

utils.py

@@ -0,0 +1,127 @@
+import os
+import logging
+import csv
+import hashlib
+from dotenv import load_dotenv
+
+import chromadb
+from llama_index.core import Settings
+from llama_index.core import Document
+from llama_index.core.schema import BaseNode
+from llama_index.core.node_parser import TokenTextSplitter
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.core.ingestion import IngestionPipeline
+from llama_index.core import VectorStoreIndex
+from llama_index.vector_stores.chroma import ChromaVectorStore
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.llms.openai import OpenAI
+
+from config import CHROMA_PATH, CHROMA_COLLECTION, FILES, CHUNK_SIZE, CHUNK_OVERLAP
+
+Settings.llm = OpenAI(temperature=0, model="gpt-4o-mini")
+Settings.embed_model = OpenAIEmbedding(
+    model="text-embedding-3-small"
+)
+
+load_dotenv()
+
+def deterministic_id_func(i: int, doc: BaseNode) -> str:
+    """Deterministic ID function for the text splitter.
+    This will be used to generate a unique repeatable identifier for each node."""
+    unique_identifier = doc.id_ + str(i)
+    hasher = hashlib.sha256()
+    hasher.update(unique_identifier.encode('utf-8')) 
+    return hasher.hexdigest()
+
+def create_db(return_nodes=False):
+    rows = []
+    # Load the file as a JSON
+    for FILE in FILES:
+        with open(FILE, mode="r", encoding="utf-8") as file:
+            csv_reader = csv.reader(file)
+
+            for idx, row in enumerate(csv_reader):
+                if idx == 0: continue # Skip header row
+                rows.append(row)
+
+    # Convert the chunks to Document objects so the LlamaIndex framework can process them.
+    documents = [Document(text=row[1], metadata={"title": row[0], "url": row[2]}) for row in rows]
+    # By default, the node/chunks ids are set to random uuids. To ensure same id's per run, we manually set them.
+    for idx, doc in enumerate(documents):
+        doc.id_ = f"doc_{idx}"
+
+    # Define the splitter object that split the text into segments with 512 tokens,
+    # with a 128 overlap between the segments.
+    text_splitter = TokenTextSplitter(
+        separator=" ", chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP,
+        id_func=deterministic_id_func
+    )
+
+    chroma_client = chromadb.PersistentClient(path=CHROMA_PATH)
+    chroma_collection = chroma_client.get_or_create_collection(CHROMA_COLLECTION)
+    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+    # Create the pipeline to apply the transformation (splitting and embedding) on each chunk,
+    # and store the transformed text in the chroma vector store.
+    pipeline = IngestionPipeline(
+        transformations=[
+            text_splitter,
+            OpenAIEmbedding(model = 'text-embedding-3-small'),
+        ],
+        vector_store=vector_store
+    )
+
+    # Run the transformation pipeline.
+    nodes = pipeline.run(documents=documents, show_progress=True)
+
+    db = chromadb.PersistentClient(path=CHROMA_PATH)
+    chroma_collection = db.get_or_create_collection(CHROMA_COLLECTION)
+    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+
+    index = VectorStoreIndex.from_vector_store(vector_store)
+    if return_nodes:
+        return nodes
+    else:
+        return index
+
+
+def load_db():
+    chroma_client = chromadb.PersistentClient(CHROMA_PATH)
+    chroma_collection = chroma_client.get_collection(CHROMA_COLLECTION)
+
+    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+
+    index = VectorStoreIndex.from_vector_store(
+        vector_store=vector_store,
+        show_progress=True,
+        use_async=True,
+        embed_model=Settings.embed_model
+    )
+
+    return index
+
+def load_asset(file):
+    """Load CSS from an external file"""
+    if os.path.exists(file):
+        with open(file, "r", encoding="utf-8") as f:
+            return f.read()
+
+def num_tokens_from_messages(messages, model="gpt-4"):
+    """Return the number of tokens used by a list of messages."""
+    try:
+        encoding = tiktoken.encoding_for_model(model)
+    except KeyError:
+        encoding = tiktoken.get_encoding("cl100k_base")
+    
+    tokens_per_message = 3
+    tokens_per_name = 1
+    
+    num_tokens = 0
+    for message in messages:
+        num_tokens += tokens_per_message
+        for key, value in message.items():
+            num_tokens += len(encoding.encode(value))
+            if key == "name":
+                num_tokens += tokens_per_name
+    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
+    
+    return num_tokens