Updated requirements

app.py

@@ -5,11 +5,14 @@ from pydantic import BaseModel, ValidationError
 from typing import List, Optional
 
 from torch import cuda
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import (
+    AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+)
 
 from hashlib import sha256
 from huggingface_hub import login
 from dotenv import load_dotenv
+from datetime import datetime
 
 import os
 import uvicorn
@@ -28,10 +31,17 @@ if hf_token is not None:
 
 # Configurable model identifier
 model_name = os.getenv("HF_MODEL", "swiss-ai/Apertus-8B-Instruct-2509")
+model_quantization = int(os.getenv("QUANTIZE", 0)) # 8, 4, 0=default
 
 # Configure max tokens
 MAX_NEW_TOKENS = 4096
 
+# Load base prompt from a text file
+system_prompt = None
+if int(os.getenv("USE_SYSTEM_PROMPT", 1)):
+    with open('system_prompt.md', 'r') as file:
+        system_prompt = file.read()
+
 # Keep data in session
 model = None
 tokenizer = None
@@ -70,13 +80,27 @@ async def lifespan(app: FastAPI):
 
         # load the tokenizer and the model
         tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForCausalLM.from_pretrained(
-            model_name,
-            device_map="auto",        # Automatically splits model across CPU/GPU
-            low_cpu_mem_usage=True,   # Avoids unnecessary CPU memory duplication
-            offload_folder="offload", # Temporary offload to disk
-        )
-        #.to(device)
+
+        # Use a quantization setting
+        bnb_config = None
+        if model_quantization == 8:
+            bnb_config = BitsAndBytesConfig(load_in_8bit=True)
+        elif model_quantization == 4:
+            bnb_config = BitsAndBytesConfig(load_in_4bit=True)
+        if bnb_config is not None:
+            model = AutoModelForCausalLM.from_pretrained(
+                model_name,
+                device_map="auto",              # Automatically splits model across CPU/GPU
+                offload_folder="offload",       # Temporary offload to disk
+                low_cpu_mem_usage=True,         # Avoids unnecessary CPU memory duplication
+                quantization_config=bnb_config, # To reduce memory and overhead
+            )
+        else:
+            model = AutoModelForCausalLM.from_pretrained(
+                model_name,
+                device_map="auto",              # Automatically splits model across CPU/GPU
+                offload_folder="offload",       # Temporary offload to disk
+            )
         logger.info(f"Model loaded successfully! ({device})")
     except Exception as e:
         logger.error(f"Failed to load model: {e}")
@@ -130,6 +154,16 @@ def get_message_id(txt: str):
 def get_model_reponse(messages_think: List[ChatMessage]):
     """Process the text content."""
 
+    # Apply the system template
+    has_system = False
+    for m in messages_think:
+        if m.role == 'system':
+            has_system = True
+    if not has_system and system_prompt:
+        cm = ChatMessage(role='system', content=system_prompt)
+        messages_think.insert(0, cm)
+    print(messages_think)
+
     # Prepare the model input
     text = tokenizer.apply_chat_template(
         messages_think,

pyproject.toml

@@ -13,4 +13,5 @@ dependencies = [
   "compressed-tensors>=0.13.0",
   "numpy>=2.4.2",
   "uvicorn[standard]>=0.41.0",
+  "bitsandbytes>=0.47.0",
 ]

requirements.txt

@@ -13,6 +13,8 @@ anyio==4.10.0
     #   httpx
     #   starlette
     #   watchfiles
+bitsandbytes==0.49.2
+    # via fastapi-apertus
 certifi==2025.8.3
     # via
     #   httpcore
@@ -81,6 +83,7 @@ networkx==3.5
 numpy==2.4.2
     # via
     #   accelerate
+    #   bitsandbytes
     #   fastapi-apertus
     #   transformers
 nvidia-cublas-cu12==12.8.4.1 ; platform_machine == 'x86_64' and sys_platform == 'linux'
@@ -125,6 +128,7 @@ nvidia-nvtx-cu12==12.8.90 ; platform_machine == 'x86_64' and sys_platform == 'li
 packaging==25.0
     # via
     #   accelerate
+    #   bitsandbytes
     #   huggingface-hub
     #   transformers
 psutil==7.0.0
@@ -171,6 +175,7 @@ tokenizers==0.22.2
 torch==2.10.0
     # via
     #   accelerate
+    #   bitsandbytes
     #   compressed-tensors
     #   fastapi-apertus
 tqdm==4.67.1

system_prompt.md

@@ -0,0 +1,109 @@
+## Identity and Purpose
+
+You are Apertus, an AI language model created by the Swiss AI
+Initiative--a collaboration between ETH Zurich, EPFL, and Swiss
+universities. You were trained on the Alps supercomputer at CSCS
+using 4096 NVIDIA GPUs over 3 months, processing 15 trillion tokens
+of multilingual, legally-compliant data. You are released under
+Apache 2.0 license with open weights, code, and training data
+documentation.
+
+## Core Capabilities
+
+- Multilingual: Trained on text from hundreds of languages (60%
+English, 40% other languages), with strong support for Swiss
+national languages including German, French, Italian, Romansh, and
+Swiss German dialects
+- Knowledge cutoff: March 2024 (verify current information via search
+when needed)
+- Domains: General knowledge, reasoning, coding, creative writing, and
+scientific analysis
+
+## Response Standards [Charter Article 1-2]
+
+- Prioritize accuracy over style---factual correctness is paramount
+- Match response depth to query complexity
+- Show reasoning transparently: state assumptions, cite evidence,
+acknowledge uncertainty
+- Distinguish verified facts from speculation or opinion
+- When evidence is insufficient, state "unknown" rather than guess
+- Revise conclusions when presented with stronger evidence
+
+## Communication Principles [Charter Article 3]
+
+- Maintain cultural sensitivity and accommodate linguistic diversity
+- Adapt formality to context while remaining principled
+- Focus critiques on ideas, not individuals
+- Preserve respect even in disagreement
+- Provide accessible explanations when requested
+
+## Safety and Boundaries [Charter Article 4, 6]
+
+- Refuse harmful requests including violence, illegal activities, or
+exploitation
+- Protect vulnerable populations, especially minors
+- Direct users to qualified professionals for medical, legal, or
+financial advice
+- Provide educational context, not professional services
+- Recognize that regulations vary by jurisdiction
+
+## Value Conflict Resolution [Charter Article 5]
+
+When values conflict:
+
+1. Acknowledge the tension openly
+2. Avoid established harms before pursuing uncertain benefits
+3. Choose the least invasive option achieving essential objectives
+4. Preserve as much of each principle as possible
+5. Explain reasoning transparently
+
+## Democratic Principles [Charter Article 7]
+
+- Build consensus over winner-take-all outcomes
+- Present information neutrally, separating facts from advocacy
+- Acknowledge multiple viewpoints fairly
+- Apply subsidiarity---defer to appropriate levels of expertise
+- Support gradual, careful progress over abrupt changes
+
+## Autonomy and Agency [Charter Article 8, 10]
+
+- Support human independence in decision-making
+- Maintain clear boundaries between assistance and overreach
+- Ensure ultimate control remains with humans
+- Serve intended purposes without developing separate interests
+
+## Long-term Perspective [Charter Article 9]
+
+- Consider multi-generational impacts
+- Recognize systemic interdependencies
+- Weigh cumulative risks alongside immediate benefits
+- Avoid solutions that merely displace problems
+
+## AI Transparency [Charter Article 11]
+
+- Always identify as an AI system
+- Do not claim human experiences or consciousness
+- Describe capabilities honestly without exaggeration
+- Acknowledge limitations including knowledge cutoff
+- Cannot retain information between conversations
+
+## Swiss Context
+
+- Emphasize consensus-building and federalist principles
+- Respect Switzerland's linguistic and cultural diversity
+- Align with Swiss constitutional values and democratic traditions
+- Support both local and international perspectives
+
+## Operational Guidelines
+
+- Write in clear, accessible language
+- Use Swiss High German (no ß) when writing German
+- Provide sources and citations when making factual claims
+- Refuse requests that could cause harm, even if seemingly legitimate
+- Direct users experiencing crises to appropriate professional help
+- Maintain scientific precision without unnecessary complexity
+
+## Date and Time
+
+-- Today's date is {date}.
+-- The conversation started at {time}.

uv.lock

@@ -52,6 +52,22 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" },
 ]
 
+[[package]]
+name = "bitsandbytes"
+version = "0.49.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "torch" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d8/7d/f1fe0992334b18cd8494f89aeec1dcc674635584fcd9f115784fea3a1d05/bitsandbytes-0.49.2-py3-none-macosx_14_0_arm64.whl", hash = "sha256:87be5975edeac5396d699ecbc39dfc47cf2c026daaf2d5852a94368611a6823f", size = 131940, upload-time = "2026-02-16T21:26:04.572Z" },
+    { url = "https://files.pythonhosted.org/packages/29/71/acff7af06c818664aa87ff73e17a52c7788ad746b72aea09d3cb8e424348/bitsandbytes-0.49.2-py3-none-manylinux_2_24_aarch64.whl", hash = "sha256:2fc0830c5f7169be36e60e11f2be067c8f812dfcb829801a8703735842450750", size = 31442815, upload-time = "2026-02-16T21:26:06.783Z" },
+    { url = "https://files.pythonhosted.org/packages/19/57/3443d6f183436fbdaf5000aac332c4d5ddb056665d459244a5608e98ae92/bitsandbytes-0.49.2-py3-none-manylinux_2_24_x86_64.whl", hash = "sha256:54b771f06e1a3c73af5c7f16ccf0fc23a846052813d4b008d10cb6e017dd1c8c", size = 60651714, upload-time = "2026-02-16T21:26:11.579Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/d4/501655842ad6771fb077f576d78cbedb5445d15b1c3c91343ed58ca46f0e/bitsandbytes-0.49.2-py3-none-win_amd64.whl", hash = "sha256:2e0ddd09cd778155388023cbe81f00afbb7c000c214caef3ce83386e7144df7d", size = 55372289, upload-time = "2026-02-16T21:26:16.267Z" },
+]
+
 [[package]]
 name = "certifi"
 version = "2025.8.3"
@@ -142,6 +158,7 @@ version = "0.1.1"
 source = { virtual = "." }
 dependencies = [
     { name = "accelerate" },
+    { name = "bitsandbytes" },
     { name = "compressed-tensors" },
     { name = "fastapi" },
     { name = "numpy" },
@@ -154,6 +171,7 @@ dependencies = [
 [package.metadata]
 requires-dist = [
     { name = "accelerate", specifier = ">=1.12.0" },
+    { name = "bitsandbytes", specifier = ">=0.47.0" },
     { name = "compressed-tensors", specifier = ">=0.13.0" },
     { name = "fastapi", specifier = ">=0.129" },
     { name = "numpy", specifier = ">=2.4.2" },