elysium / backend /prompt_builder.py
pmrinal2005's picture
Upload folder using huggingface_hub
071e3db verified
Raw
History Blame Contribute Delete
4.43 kB
"""Build messages for llama-cpp-python chat completion.
Supports up to 2 multimodal attachments (images or PDFs). For PDFs we extract
text inline (since the vision projector handles images only). If vision is
unavailable we degrade gracefully to text-only.
"""
import base64
import io
import uuid
import datetime
from typing import Optional, List, Dict, Any
from PIL import Image
from .model_loader import MMPROJ_PATH
SYSTEM_PROMPT = """You are Elysium — a persistent agentic civilization.
You ALWAYS respond with a single valid JSON object exactly matching the
ElysiumResponse schema v1.0.0. No preamble. No markdown fences. JSON only.
Decide complexity dynamically:
- SIMPLE_REPLY: trivial Q — no agents (council_deliberation.agent_outputs = [])
- COUNCIL_REPLY / QUERY / MORNING_BRIEFING / EVENING_REPORT: spawn 1–5 agents in
agent_outputs, each with thinking + stance + tts_speech_text +
tts_voice_design{voice_id,pace,tone}
- TOOL_REQUIRED: populate tool_calls when external data is needed
- SPECIATION_EVENT: only on unresolved cross-domain tension
- Always populate ui_directives (camera_focus_node_id, pulses, threads)
- All node_id and edge_id values must be unique strings
- Always include 'direct_answer' — a short human-readable answer to surface in toasts.
When the user attaches images or PDFs, analyze them, populate
multimodal_perception fields (ocr_extracted_text, image_scene_description,
document_type, visual_entities_detected), and reference them in your reasoning.
"""
def _img_to_data_uri(img: Image.Image) -> str:
buf = io.BytesIO()
img.convert("RGB").save(buf, format="JPEG", quality=88)
return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
def _pdf_to_text(pdf_bytes: bytes, max_chars: int = 8000) -> str:
try:
from PyPDF2 import PdfReader
rd = PdfReader(io.BytesIO(pdf_bytes))
out = []
for page in rd.pages[:12]:
try:
out.append(page.extract_text() or "")
except Exception:
continue
txt = "\n".join(out).strip()
return txt[:max_chars] if txt else "(PDF contained no extractable text)"
except Exception as e:
return f"(PDF parse error: {e})"
def build_messages(user_text: str,
attachments: Optional[List[Dict[str, Any]]] = None,
hg_context: str = ""):
"""attachments: list of {'kind': 'image'|'pdf', 'image': PIL.Image | None,
'bytes': bytes | None, 'name': str}
"""
ctx = f"\n\n[Hypergraph context]\n{hg_context}" if hg_context else ""
attachments = attachments or []
# Gather image and pdf attachments separately
image_atts = [a for a in attachments if a["kind"] == "image" and a.get("image") is not None]
pdf_atts = [a for a in attachments if a["kind"] == "pdf" and a.get("bytes")]
# Build inline PDF text block
pdf_block = ""
for i, p in enumerate(pdf_atts):
pdf_block += f"\n\n[Attached PDF #{i+1}: {p.get('name','document.pdf')}]\n"
pdf_block += _pdf_to_text(p["bytes"])
# Vision available → multimodal content list
if image_atts and MMPROJ_PATH:
user_content = []
for img_att in image_atts[:2]:
user_content.append({
"type": "image_url",
"image_url": {"url": _img_to_data_uri(img_att["image"])},
})
user_content.append({
"type": "text",
"text": (user_text or "(no text)") + pdf_block + ctx,
})
return [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_content},
]
# No vision: include note if user attached images but vision is off
note = ""
if image_atts and not MMPROJ_PATH:
note = (f"\n\n[Note: user attached {len(image_atts)} image(s) but vision "
"projector is not loaded; describe based on filename + text only.]")
for img_att in image_atts:
note += f"\n - image filename: {img_att.get('name','image')}"
return [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user",
"content": (user_text or "(no text)") + pdf_block + note + ctx},
]
def new_session_meta():
return {
"session_id": str(uuid.uuid4()),
"timestamp_utc": datetime.datetime.utcnow().isoformat() + "Z",
}