Upload 3 files

app/app.py

@@ -0,0 +1,189 @@
+import chainlit as cl
+from openai import AuthenticationError
+
+from aicore.logger import _logger
+from aicore.config import LlmConfig
+from aicore.const import STREAM_END_TOKEN, STREAM_START_TOKEN, REASONING_START_TOKEN, REASONING_STOP_TOKEN
+
+from aicore.llm import Llm
+
+from ulid import ulid
+import asyncio
+import time
+
+from utils import MODELS_PROVIDERS_MAP, PROVIDERS_API_KEYS, REASONER_PROVIDERS_MAP, check_openai_api_key, trim_messages
+from settings import PROFILES_SETTINGS
+
+DEFAULT_REASONER_CONFIG = LlmConfig(
+    provider="groq",
+    api_key=PROVIDERS_API_KEYS.get("groq"),
+    model="deepseek-r1-distill-llama-70b",
+    temperature=0.5,
+    max_tokens=1024
+)
+
+DEFAULT_LLM_CONFIG = {
+    "Reasoner4All": LlmConfig(
+        provider="groq",
+        api_key=PROVIDERS_API_KEYS.get("groq"),
+        model="meta-llama/llama-4-scout-17b-16e-instruct",
+        temperature=0,
+        max_tokens=4196,
+        # reasoner=DEFAULT_REASONER_CONFIG
+    ),
+    "OpenAi": LlmConfig(
+        provider="openai",
+        api_key=PROVIDERS_API_KEYS.get("openai", ""),
+        model="gpt-4o-mini",
+        temperature=0,
+        max_tokens=4196,
+        # reasoner=DEFAULT_REASONER_CONFIG
+    )
+}
+
+@cl.set_chat_profiles
+async def chat_profile():
+    return [
+        cl.ChatProfile(
+            name="Reasoner4All",
+            markdown_description="Talk with the lastest Llm models! Powered by AiCore, check it on GitHub, link in Readme",
+            icon="https://picsum.photos/200",
+        ),
+        cl.ChatProfile(
+            name="OpenAi",
+            markdown_description="Talk with the lastest Llm models! Powered by AiCore, check it on GitHub, link in Readme",
+            icon="https://picsum.photos/200",
+        )
+    ]
+    
+@cl.on_settings_update
+async def setup_agent(settings):
+    provider = MODELS_PROVIDERS_MAP.get(settings.get("Model"), "openai")
+    llm_config = LlmConfig(
+        provider=provider,
+        api_key=PROVIDERS_API_KEYS.get(provider) or settings.get("Api Key"),
+        model=settings.get("Model"),
+        temperature=settings.get("Temperature"),
+        max_tokens=settings.get("Max Tokens")
+    )
+    if settings.get("Use Reasoner"):
+        reasoner_provder = REASONER_PROVIDERS_MAP.get(settings.get("Reasoner Model"), "openai")
+        reasoner_config = LlmConfig(
+            provider=reasoner_provder,
+            api_key=PROVIDERS_API_KEYS.get(reasoner_provder) or settings.get("Reasoner Api Key"),
+            model=settings.get("Reasoner Model"),
+            temperature=settings.get("Reasoner Temperature"),
+            max_tokens=settings.get("Reasoner Max Tokens")
+        )
+        llm_config.reasoner = reasoner_config
+    
+    llm = Llm.from_config(llm_config)
+    llm.session_id = ulid()
+    llm.system_prompt = settings.get("System Prompt")
+    if llm.reasoner:
+        llm.reasoner.system_prompt = settings.get("Reasoner System Prompt")
+
+    cl.user_session.set(
+        "llm", llm
+    )
+
+@cl.on_chat_start
+async def start_chat():
+    user_profile = cl.user_session.get("chat_profile")
+    cl.user_session.set("history", [])
+    llm_config = DEFAULT_LLM_CONFIG.get(user_profile)
+    llm = Llm.from_config(llm_config)
+    llm.session_id = ulid()
+    cl.user_session.set(
+        "llm", llm
+    )
+    
+    settings = await cl.ChatSettings(
+        PROFILES_SETTINGS.get(user_profile)
+    ).send()
+    
+
+async def run_concurrent_tasks(llm, message):
+    asyncio.create_task(llm.acomplete(message))
+    asyncio.create_task(_logger.distribute())
+    # Stream logger output while LLM is running
+    while True:        
+        async for chunk in _logger.get_session_logs(llm.session_id):
+            yield chunk  # Yield each chunk directly
+
+@cl.on_message
+async def main(message: cl.Message):
+    llm = cl.user_session.get("llm")
+    if not llm.config.api_key:
+        while True:
+            api_key_msg = await cl.AskUserMessage(content="Please provide a valid api_key", timeout=10).send()
+            if api_key_msg:
+                api_key = api_key_msg.get("output")
+                valid = check_openai_api_key(api_key)
+                if valid:
+                    await cl.Message(
+                        content=f"Config updated with key.",
+                    ).send()
+                    llm.config.api_key = api_key
+                    cl.user_session.set("llm", llm)
+                    break
+    
+    start = time.time()
+    thinking=False
+    
+    history = cl.user_session.get("history")
+    history.append(message.content)
+    history = trim_messages(history, llm.tokenizer)
+    model_id = None
+    try:
+        if llm.reasoner is not None or llm.config.model in REASONER_PROVIDERS_MAP:
+            # Streaming the thinking
+            async with cl.Step(name=f"{llm.reasoner.config.provider} - {llm.reasoner.config.model} to think", type="llm") as thinking_step:
+                msg = cl.Message(content="")
+                async for chunk in run_concurrent_tasks(
+                        llm,
+                        message=history
+                    ):
+                    if chunk == STREAM_START_TOKEN:
+                        continue
+                    if chunk == REASONING_START_TOKEN:
+                        thinking = True
+                        continue
+                        # chunk = " - *reasoning*\n```html\n"
+                    if chunk == REASONING_STOP_TOKEN:
+                        thinking = False
+                        thought_for = round(time.time() - start)
+                        thinking_step.name = f"{llm.reasoner.config.model} to think for {thought_for}s"
+                        await thinking_step.update()
+                        chunk = f"```{llm.config.model}```\n"
+                        model_id = f"```{llm.config.model}```\n"
+
+                    if chunk == STREAM_END_TOKEN:
+                        break
+                    
+                    if thinking:
+                        await thinking_step.stream_token(chunk)
+                    else:
+                        await msg.stream_token(chunk)
+        else:
+            msg = cl.Message(content="")
+            async for chunk in run_concurrent_tasks(
+                    llm,
+                    message=history
+                ):
+                if chunk == STREAM_START_TOKEN:
+                    continue
+
+                if chunk == STREAM_END_TOKEN:
+                        break
+                
+                await msg.stream_token(chunk)
+            
+        hst_msg = msg.content.replace(model_id, "") if model_id else msg.content
+        history.append(hst_msg)
+        await msg.send()
+    
+    except Exception as e:
+        await cl.ErrorMessage("Internal Server Error").send()
+
+### TODO add future todos, include support for images and pdf upload for conversation

app/settings.py

@@ -0,0 +1,152 @@
+from chainlit.input_widget import Select, Slider, Switch, TextInput
+from utils import MODELS_PROVIDERS_MAP, REASONER_PROVIDERS_MAP
+
+OPEN_LLM_SETTINGS = [
+    Select(
+        id="Model",
+        label="Llm",
+        values=list(MODELS_PROVIDERS_MAP.keys()),
+        initial_index=-1,
+    ),
+    Slider(
+        id="Temperature",
+        label="Temperature",
+        initial=0,
+        min=0,
+        max=1,
+        step=0.1,
+    ),
+    Slider(
+        id="Max Tokens",
+        label="Max Tokens",
+        initial=4196,
+        min=512,
+        max=8192,
+        step=256,
+    ),
+    TextInput(
+        id="System Prompt",
+        label="System Prompt",
+        initial="You are a helpful assistant."
+    ), 
+    Switch(
+        id="Use Reasoner",
+        label="Use reasoner model",
+        initial=False
+    )
+]
+
+REASONER_SETTINGS = [
+    Select(
+        id="Reasoner Model",
+        label="Reasoner Llm",
+        values=list(REASONER_PROVIDERS_MAP.keys()),
+        initial_index=0,
+    ),
+    Slider(
+        id="Reasoner Temperature",
+        label="Reasoner Temperature",
+        initial=0.5,
+        min=0,
+        max=1,
+        step=0.1,
+    ),
+    Slider(
+        id="Reasoner Max Tokens",
+        label="Reasoner Max Tokens",
+        initial=1024,
+        min=512,
+        max=4096,
+        step=256,
+    ),
+    TextInput(
+        id="Reasoner System Prompt",
+        label="Reasoner System Prompt",
+        initial="You are a helpfull assistant with reasoning capabilites that breaks down problems into the detailed steps required to solve them"
+    ), 
+]
+
+
+OPEN_AI_REASONER_SETTINGS = [
+    Select(
+        id="Reasoner Model",
+        label="Reasoner Llm",
+        values=list(REASONER_PROVIDERS_MAP.keys()),
+        initial_index=0,
+    ),
+    TextInput(
+        id="Reasoner Api Key",
+        label="Reasoner Api Key",
+        initial="your api key goes here..."
+    ),
+    Slider(
+        id="Reasoner Temperature",
+        label="Reasoner Temperature",
+        initial=0.5,
+        min=0,
+        max=1,
+        step=0.1,
+    ),
+    Slider(
+        id="Reasoner Max Tokens",
+        label="Reasoner Max Tokens",
+        initial=1024,
+        min=512,
+        max=4096,
+        step=256,
+    ),
+    TextInput(
+        id="Reasoner System Prompt",
+        label="Reasoner System Prompt",
+        initial="You are a helpfull assistant with reasoning capabilites that breaks down problems into the detailed steps required to solve them"
+    ), 
+]
+
+OPEN_AI_SETTINGS = [
+    TextInput(
+        id="Model",
+        label="Llm",
+        initial="gpt-4o-mini"
+    ),
+    TextInput(
+        id="Api Key",
+        label="Api Key",
+        initial="your api key goes here..."
+    ),
+    TextInput(
+        id="Base Url",
+        label="Base Url",
+        initial="leave this empty to connect to chatGPT"
+    ),
+    Slider(
+        id="Temperature",
+        label="Temperature",
+        initial=0,
+        min=0,
+        max=1,
+        step=0.1,
+    ),
+    Slider(
+        id="Max Tokens",
+        label="Max Tokens",
+        initial=4196,
+        min=512,
+        max=8192,
+        step=256,
+    ),
+    TextInput(
+        id="System Prompt",
+        label="System Prompt",
+        initial="You are a helpful assistant."
+    ),     
+    Switch(
+        id="Use Reasoner",
+        label="Use reasoner model",
+        initial=False
+    )
+]
+
+PROFILES_SETTINGS = {
+    "Reasoner4All": OPEN_LLM_SETTINGS + REASONER_SETTINGS,
+    "OpenAi": OPEN_AI_SETTINGS + OPEN_AI_REASONER_SETTINGS
+}

app/utils.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+from typing import Optional
+import openai
+import os
+
+CONFIG_DIR = Path("./config")
+
+REASONER_PROVIDERS_MAP = {
+    "deepseek-r1-distill-llama-70b": "groq",
+    "deepseek-ai/deepseek-r1": "nvidia",
+    "deepseek/deepseek-r1:free": "openrouter"
+}
+
+MODELS_PROVIDERS_MAP = {
+    "mistral-small-latest": "mistral",
+    "open-mistral-nemo": "mistral",
+    "mistral-large-latest": "mistral",
+    "gemini-2.5-pro-exp-03-25": "gemini",
+    "gemini-2.0-flash-thinking-exp-01-21": "gemini",
+    "gemini-2.0-flash-exp": "gemini",
+    "qwen-2.5-32b": "groq",
+    "qwen-2.5-coder-32b": "groq",
+    "qwen-qwq-32b": "groq",
+    "gemma2-9b-it": "groq",
+    "llama-3.3-70b-versatile": "groq",
+    "llama-3.2-3b-preview": "groq",
+    "meta-llama/llama-4-scout-17b-16e-instruct": "groq"
+}
+
+PROVIDERS_API_KEYS = {
+    "gemini": os.environ.get("GEMINI_API_KEY"),
+    "groq": os.environ.get("GROQ_API_KEY"),
+    "mistral": os.environ.get("MISTRAL_API_KEY"),
+    "nvidia": os.environ.get("NVIDIA_API_KEY"),
+    "openrouter": os.environ.get("OPENROUTER_API_KEY")
+}
+
+def check_openai_api_key(api_key, base_url=None):
+    client = openai.OpenAI(api_key=api_key, base_url=base_url)
+    try:
+        client.models.list()
+    except openai.AuthenticationError:
+        return False
+    else:
+        return True
+
+def trim_messages(messages, tokenizer_fn, max_tokens :Optional[int]=None):
+    max_tokens = max_tokens or int(os.environ.get("MAX_HISTORY_TOKENS", 1028))
+    while messages and sum(len(tokenizer_fn(msg)) for msg in messages) > max_tokens:
+        messages.pop(0)  # Remove from the beginning
+    return messages