File size: 3,659 Bytes
c9622da |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
"""Phi-3.5-mini model wrapper using llama-cpp-python."""
from typing import Optional, List, Dict, Any
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
from src.config import ModelConfig
class PhiModel:
"""Wrapper for Phi-3.5-mini model."""
def __init__(self, config: Optional[ModelConfig] = None):
"""Initialize the model wrapper.
Args:
config: Model configuration. Uses defaults if not provided.
"""
self.config = config or ModelConfig()
self._model: Optional[Llama] = None
self._model_path: Optional[str] = None
@property
def model(self) -> Llama:
"""Lazy load the model."""
if self._model is None:
self._load_model()
return self._model
def _load_model(self) -> None:
"""Download and load the model."""
print(f"Downloading model from {self.config.repo_id}...")
self._model_path = hf_hub_download(
repo_id=self.config.repo_id,
filename=self.config.filename
)
print("Loading model into memory...")
self._model = Llama(
model_path=self._model_path,
n_ctx=self.config.n_ctx,
n_threads=self.config.n_threads,
verbose=self.config.verbose
)
print("Model loaded successfully!")
def generate(self, prompt: str, max_tokens: Optional[int] = None) -> str:
"""Generate text completion.
Args:
prompt: Input prompt.
max_tokens: Maximum tokens to generate.
Returns:
Generated text.
"""
output = self.model(
prompt,
max_tokens=max_tokens or self.config.max_tokens,
temperature=self.config.temperature,
echo=False
)
return output["choices"][0]["text"].strip()
def chat(
self,
messages: List[Dict[str, str]],
max_tokens: Optional[int] = None
) -> str:
"""Generate chat completion.
Args:
messages: List of message dicts with 'role' and 'content'.
max_tokens: Maximum tokens to generate.
Returns:
Assistant's response.
"""
output = self.model.create_chat_completion(
messages=messages,
max_tokens=max_tokens or self.config.max_tokens,
temperature=self.config.temperature
)
return output["choices"][0]["message"]["content"].strip()
def chat_with_context(
self,
query: str,
context: str,
system_prompt: Optional[str] = None
) -> str:
"""Generate response with RAG context.
Args:
query: User's question.
context: Retrieved context from documents.
system_prompt: Optional system prompt.
Returns:
Generated response.
"""
if system_prompt is None:
system_prompt = (
"You are a helpful assistant. Answer the user's question based on "
"the provided context. If the context doesn't contain relevant "
"information, say so honestly. Be concise and accurate."
)
user_message = f"""Context:
{context}
Question: {query}
Please answer based on the context provided above."""
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_message}
]
return self.chat(messages)
|