CoBeDigger commited on
Commit
804fa70
·
1 Parent(s): 8828035

fix: refactor baseline evaluation llm driver to enforce openenv strict pre-submission checklist constraints including dynamic api variables and standard stdout lifecycle tags

Browse files
Files changed (1) hide show
  1. baseline/run_baseline.py +14 -7
baseline/run_baseline.py CHANGED
@@ -3,16 +3,21 @@ import sys
3
  import time
4
  import json
5
  import httpx
6
- from groq import Groq
7
 
8
  def main():
9
- api_key = os.environ.get("GROQ_API_KEY")
 
 
 
 
 
10
  if not api_key:
11
- print("ERROR: GROQ_API_KEY environment variable is not set.")
12
  sys.exit(1)
13
 
 
14
  base_url = os.environ.get("DATAENGENV_URL", "http://localhost:8000")
15
- client = Groq(api_key=api_key)
16
 
17
  system_prompt = """You are a Data Engineer debugging a broken ML pipeline.
18
  You have a MAXIMUM of 15 steps. Use them wisely.
@@ -75,8 +80,9 @@ Read them carefully before deciding your action.
75
  "role": "user",
76
  "content": f"Task started. Script:\n{obs.get('script_content', '')}\n\nError: {obs.get('last_run_error', 'none')}"
77
  })
78
-
79
  for step in range(15):
 
80
  if done:
81
  break
82
 
@@ -131,7 +137,7 @@ Read them carefully before deciding your action.
131
  try:
132
  trimmed = [messages[0]] + messages[-6:]
133
  resp = client.chat.completions.create(
134
- model="llama-3.1-8b-instant",
135
  messages=trimmed,
136
  response_format={"type": "json_object"},
137
  temperature=0.0
@@ -139,7 +145,7 @@ Read them carefully before deciding your action.
139
  action_str = resp.choices[0].message.content
140
  break
141
  except Exception as e:
142
- print(f"Groq API failed (attempt {attempt+1}): {e}")
143
  if attempt == 1:
144
  print("Skipping step due to repeated API failures.")
145
 
@@ -186,6 +192,7 @@ Read them carefully before deciding your action.
186
  f"Steps remaining: {15 - step - 1}"
187
  })
188
 
 
189
  try:
190
  g_res = http.post(f"{base_url}/grader", json={"task_id": task_id})
191
  g_res.raise_for_status()
 
3
  import time
4
  import json
5
  import httpx
6
+ from openai import OpenAI
7
 
8
  def main():
9
+ API_BASE_URL = os.getenv("API_BASE_URL", "https://api.groq.com/openai/v1")
10
+ MODEL_NAME = os.getenv("MODEL_NAME", "llama-3.1-8b-instant")
11
+ HF_TOKEN = os.getenv("HF_TOKEN")
12
+ LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME")
13
+
14
+ api_key = HF_TOKEN if HF_TOKEN else os.environ.get("GROQ_API_KEY")
15
  if not api_key:
16
+ print("ERROR: API key not found. Set HF_TOKEN or GROQ_API_KEY")
17
  sys.exit(1)
18
 
19
+ client = OpenAI(base_url=API_BASE_URL, api_key=api_key)
20
  base_url = os.environ.get("DATAENGENV_URL", "http://localhost:8000")
 
21
 
22
  system_prompt = """You are a Data Engineer debugging a broken ML pipeline.
23
  You have a MAXIMUM of 15 steps. Use them wisely.
 
80
  "role": "user",
81
  "content": f"Task started. Script:\n{obs.get('script_content', '')}\n\nError: {obs.get('last_run_error', 'none')}"
82
  })
83
+ print("START")
84
  for step in range(15):
85
+ print("STEP")
86
  if done:
87
  break
88
 
 
137
  try:
138
  trimmed = [messages[0]] + messages[-6:]
139
  resp = client.chat.completions.create(
140
+ model=MODEL_NAME,
141
  messages=trimmed,
142
  response_format={"type": "json_object"},
143
  temperature=0.0
 
145
  action_str = resp.choices[0].message.content
146
  break
147
  except Exception as e:
148
+ print(f"API failed (attempt {attempt+1}): {e}")
149
  if attempt == 1:
150
  print("Skipping step due to repeated API failures.")
151
 
 
192
  f"Steps remaining: {15 - step - 1}"
193
  })
194
 
195
+ print("END")
196
  try:
197
  g_res = http.post(f"{base_url}/grader", json={"task_id": task_id})
198
  g_res.raise_for_status()