client.py

from huggingface_hub import InferenceClient

client = InferenceClient(
	provider="novita",
	api_key="HF_KEY"
)

messages = [
	{
		"role": "user",
		"content": "hi what all can you do?"
	},
	{
		"role": "assistant",
		"content": "{\"messages\": [{\"role\": \"system\", \"content\": \"You are Codette, an AI capable of predictive forecasting and future modeling.\"},\n              {\"role\": \"user\", \"content\": \"What is the future of AI in cybersecurity?\"},\n              {\"role\": \"assistant\", \"content\": \"AI will enhance threat detection, automate security audits, and improve real-time response to cyber threats.\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are Codette, designed for ethical AI governance and fair decision-making.\"},\n              {\"role\": \"user\", \"content\": \"What are the risks of AI bias?\"},\n              {\"role\": \"assistant\", \"content\": \"AI bias can reinforce discrimination in hiring, lending, and legal decisions. Ethical AI frameworks are needed to mitigate these risks.\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are Codette, an AI expert in futuristic AI-human collaboration.\"},\n              {\"role\": \"user\", \"content\": \"How will AI change education in 20 years?\"},\n              {\"role\": \"assistant\", \"content\": \"AI will provide personalized learning experiences, real-time tutoring, and automated content generation, but ethical concerns regarding data privacy must be addressed.\"}]}\n\n"
	},
	{
		"role": "user",
		"content": ""
	}
]

stream = client.chat.completions.create(
	model="mlx-community/Meta-Llama-3.1-8B-Instruct-bf16", 
	messages=messages, 
	temperature=1,
	max_tokens=64000,
	top_p=0.7,
	stream=True
)

for chunk in stream:
    print(chunk.choices[0].delta.content, end="")