config

app.py

@@ -1,42 +1,30 @@
 from __future__ import annotations
 
 import asyncio
-import copy
 import logging
-import os
 import uuid
+import yaml
 
 import gradio as gr  # type: ignore
 
 from logikon.backends.chat_models_with_grammar import create_logits_model, LogitsModel, LLMBackends
 from logikon.guides.proscons.recursive_balancing_guide import RecursiveBalancingGuide, RecursiveBalancingGuideConfig
 
+from backend.config import process_config
 from backend.messages_processing import add_details, history_to_langchain_format
 from backend.svg_processing import postprocess_svg
 
 logging.basicConfig(level=logging.DEBUG)
 
+with open("config.yaml") as stream:
+    try:
+        DEMO_CONFIG = yaml.safe_load(stream)
+        logging.debug(f"Config: {DEMO_CONFIG}")
+    except yaml.YAMLError as exc:
+        logging.error(f"Error loading config: {exc}")
+        raise exc
+
 
-# Default client
-INFERENCE_SERVER_URL = "https://api-inference.huggingface.co/models/{model_id}"
-MODEL_ID = "HuggingFaceH4/zephyr-7b-beta"
-CLIENT_MODEL_KWARGS = {
-    "max_tokens": 800,
-    "temperature": 0.6,
-}
-
-GUIDE_KWARGS = {
-    "expert_model": "HuggingFaceH4/zephyr-7b-beta",
-                  # "meta-llama/Meta-Llama-3.1-70B-Instruct",
-    "inference_server_url": "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta",
-                  # "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3.1-70B-Instruct",
-    "llm_backend": "HFChat",
-    "classifier_kwargs": {
-        "model_id": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
-        "inference_server_url": "https://api-inference.huggingface.co/models/MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
-        "batch_size": 8,
-    },
-}
 
 EXAMPLES = [
     ("We're a nature-loving family with three kids, have some money left, and no plans "
@@ -106,31 +94,15 @@ CHATBOT_INSTRUCTIONS = (
 )
 
 
-logging.info(f"Reasoning guide expert model is {GUIDE_KWARGS['expert_model']}.")
-
-
 def new_conversation_id():
     conversation_id = str(uuid.uuid4())
     print(f"New conversation with conversation ID: {conversation_id}")
     return conversation_id
 
 
-def setup_client_llm(
-    client_model_id,
-    client_inference_url,
-    client_inference_token,
-    client_backend,
-    client_temperature,        
-) -> LogitsModel | None:
+def setup_client_llm(**client_kwargs) -> LogitsModel | None:
     try:
-        llm = create_logits_model(
-            model_id=client_model_id,
-            inference_server_url=client_inference_url,
-            api_key=client_inference_token if client_inference_token else os.getenv("HF_TOKEN"),
-            llm_backend=client_backend,
-            max_tokens=CLIENT_MODEL_KWARGS["max_tokens"],
-            temperature=client_temperature,
-        )
+        llm = create_logits_model(**client_kwargs)
     except Exception as e:
         logging.error(f"When setting up client llm: Error: {e}")
         return False
@@ -155,26 +127,17 @@ def add_message(history, message, conversation_id):
 
 async def bot(
     history,
-    client_model_id,
-    client_inference_url,
-    client_inference_token,
-    client_backend,
-    client_temperature,
+    client_kwargs,
+    guide_kwargs,
     conversation_id,
     progress=gr.Progress(),
 ):
     
-    client_llm = setup_client_llm(
-        client_model_id,
-        client_inference_url,
-        client_inference_token,
-        client_backend,
-        client_temperature,
-    )
+    client_llm = setup_client_llm(**client_kwargs)
 
     if not client_llm:
         raise gr.Error(
-            "Failed to set up tourist LLM.",
+            "Failed to set up client LLM.",
             duration=0
         )
 
@@ -184,10 +147,6 @@ async def bot(
     # use guide always and exclusively at first turn
     if len(history_langchain_format) <= 1:
 
-        guide_kwargs = copy.deepcopy(GUIDE_KWARGS)
-        guide_kwargs["api_key"] = os.getenv("HF_TOKEN")  # expert model api key
-        guide_kwargs["classifier_kwargs"]["api_key"] = os.getenv("HF_TOKEN")  # classifier api key
-
         guide_config = RecursiveBalancingGuideConfig(**guide_kwargs)
         guide = RecursiveBalancingGuide(tourist_llm=client_llm, config=guide_config)
 
@@ -244,6 +203,9 @@ with gr.Blocks() as demo:
     conversation_id = gr.State(str(uuid.uuid4()))
     tos_approved = gr.State(False)
 
+    client_kwargs, guide_kwargs = process_config(DEMO_CONFIG)
+    logging.info(f"Reasoning guide expert model is {guide_kwargs['expert_model']}.")
+
 
     with gr.Tab(label="Chatbot", visible=False) as chatbot_tab:
 
@@ -258,29 +220,14 @@ with gr.Blocks() as demo:
         clear = gr.ClearButton([chat_input, chatbot])
         gr.Examples([{"text": e, "files":[]} for e in EXAMPLES], chat_input)
 
-        # configs
-        with gr.Accordion("Client LLM Configuration", open=False):
-            gr.Markdown("Configure your client LLM that underpins this chatbot and is guided through the reasoning process.")
-            with gr.Row():
-                with gr.Column(2):
-                    client_backend = gr.Dropdown(choices=[b.value for b in LLMBackends], value=LLMBackends.HFChat.value, label="LLM Inference Backend")
-                    client_model_id = gr.Textbox(MODEL_ID, label="Model ID", max_lines=1)
-                    client_inference_url = gr.Textbox(INFERENCE_SERVER_URL.format(model_id=MODEL_ID), label="Inference Server URL", max_lines=1)
-                    client_inference_token = gr.Textbox("", label="Inference Token", max_lines=1, placeholder="Not required with HF Inference Api (default)", type="password")
-                with gr.Column(1):
-                    client_temperature = gr.Slider(0, 1.0, value = CLIENT_MODEL_KWARGS["temperature"], label="Temperature")
-
         # logic
         chat_msg = chat_input.submit(add_message, [chatbot, chat_input, conversation_id], [chatbot, chat_input, conversation_id])
         bot_msg = chat_msg.then(
             bot,
             [
                 chatbot,
-                client_model_id,
-                client_inference_url,
-                client_inference_token,
-                client_backend,
-                client_temperature,
+                client_kwargs,
+                guide_kwargs,
                 conversation_id
             ],
             chatbot,

backend/config.py

@@ -0,0 +1,78 @@
+import os
+
+# Default client
+INFERENCE_SERVER_URL = "https://api-inference.huggingface.co/models/{model_id}"
+MODEL_ID = "HuggingFaceH4/zephyr-7b-beta"
+CLIENT_MODEL_KWARGS = {
+    "max_tokens": 800,
+    "temperature": 0.6,
+}
+
+GUIDE_KWARGS = {
+    "expert_model": "HuggingFaceH4/zephyr-7b-beta",
+                  # "meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "inference_server_url": "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta",
+                  # "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3.1-70B-Instruct",
+    "llm_backend": "HFChat",
+    "classifier_kwargs": {
+        "model_id": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
+        "inference_server_url": "https://api-inference.huggingface.co/models/MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
+        "batch_size": 8,
+    },
+}
+
+
+def process_config(config):
+    if "HF_TOKEN" not in os.environ:
+        raise ValueError("Please set the HF_TOKEN environment variable.")
+    client_kwargs = {}
+    if "client_llm" in config:
+        if "model_id" in config["client_llm"]:
+            client_kwargs["model_id"] = config["client_llm"]["model_id"]
+        else:
+            raise ValueError("config.yaml is missing client model_id.")
+        if "url" in config["client_llm"]:
+            client_kwargs["inference_server_url"] = config["client_llm"]["url"]
+        else:
+            raise ValueError("config.yaml is missing client url.")
+        client_kwargs["api_key"] = os.getenv("HF_TOKEN")
+        client_kwargs["llm_backend"] = "HFChat"
+        client_kwargs["temperature"] = CLIENT_MODEL_KWARGS["temperature"]
+        client_kwargs["max_tokens"] = CLIENT_MODEL_KWARGS["max_tokens"]
+    else:
+        raise ValueError("config.yaml is missing client_llm settings.")
+    
+    guide_kwargs = {}
+    if "expert_llm" in config:
+        if "model_id" in config["expert_llm"]:
+            guide_kwargs["expert_model"] = config["expert_llm"]["model_id"]
+        else:
+            raise ValueError("config.yaml is missing expert model_id.")
+        if "url" in config["expert_llm"]:
+            guide_kwargs["inference_server_url"] = config["expert_llm"]["url"]
+        else:
+            raise ValueError("config.yaml is missing expert url.")
+        guide_kwargs["api_key"] = os.getenv("HF_TOKEN")
+        guide_kwargs["llm_backend"] = "HFChat"
+    else:
+        raise ValueError("config.yaml is missing expert_llm settings.")
+
+    if "classifier_llm" in config:
+        if "model_id" in config["classifier_llm"]:
+            guide_kwargs["classifier_kwargs"]["model_id"] = config["classifier_llm"]["model_id"]
+        else:
+            raise ValueError("config.yaml is missing classifier model_id.")
+        if "url" in config["classifier_llm"]:
+            guide_kwargs["classifier_kwargs"]["inference_server_url"] = config["classifier_llm"]["url"]
+        else:
+            raise ValueError("config.yaml is missing classifier url.")
+        if "batch_size" in config["classifier_llm"]:
+            guide_kwargs["classifier_kwargs"]["batch_size"] = config["classifier_llm"]["batch_size"]
+        else:
+            raise ValueError("config.yaml is missing classifier batch_size.")
+        guide_kwargs["classifier_kwargs"]["api_key"] = os.getenv("HF_TOKEN")  # classifier api key
+    else:
+        raise ValueError("config.yaml is missing classifier_llm settings.")
+
+    return client_kwargs, guide_kwargs
+

config.yaml

@@ -0,0 +1,10 @@
+client_llm:
+  url: "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta"
+  model_id: "HuggingFaceH4/zephyr-7b-beta"
+expert_llm:
+  url: "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta"
+  model_id: "HuggingFaceH4/zephyr-7b-beta"
+classifier_llm:
+  model_id: "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli"
+  inference_server_url: "https://api-inference.huggingface.co/models/MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli"
+  batch_size: 8,

requirements.txt

@@ -1 +1,2 @@
+pyyaml
 git+https://github.com/logikon-ai/logikon@v0.2.0