File size: 1,256 Bytes
1f7ead8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
from huggingface_hub import InferenceClient
from backend.app.config import config
def verify_direct():
print(f"Verifying Direct Inference with {config.LLM_MODEL}...")
prompt = (
"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"
"You are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n"
"Say 'Direct Inference Works!'<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
)
try:
client = InferenceClient(token=config.HUGGINGFACE_API_KEY)
res = client.text_generation(
prompt,
model=config.LLM_MODEL,
max_new_tokens=50,
temperature=0.7,
top_p=0.9,
stop_sequences=["<|eot_id|>"]
)
print("Success!")
print(res.strip())
except Exception as e:
msg = f"Failed: {repr(e)}"
print(msg)
with open("verify.log", "w") as f:
f.write(msg)
# Also log success
with open("verify.log", "a") as f:
f.write("\nRan verification.")
verify_direct()
|