perf: strip definitions from prompt token context to clear 429 rate limit errors

app.py

@@ -9,14 +9,15 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain_openai import ChatOpenAI
 
 # ==========================================================
-# 1. DATA INGESTION & NOTEBOOK INDEX GENERATOR
+# 1. LOAD DATA & OPTIMIZED INDEX GENERATOR (REDUCES TOKENS)
 # ==========================================================
 with open("gcmd_hierarchy.json", "r") as f:
     gcmd_data = json.load(f)
 
 def build_gcmd_indices(gcmd_json):
     """
-    Parses the GCMD hierarchy exactly as done in the working notebook.
+    Parses the GCMD hierarchy exactly like the notebook but strips 
+    heavy definitions to prevent 429 Token Rate Limit errors.
     """
     topic_list = []
     sub_tree_indices = {}
@@ -36,13 +37,9 @@ def build_gcmd_indices(gcmd_json):
             # Append current node name to path
             node_path = f"{current_path} > {node_name}" if current_path else node_name
             
-            # If it's a Variable or leaf term with text context, preserve it
+            # If it's a Variable or leaf term, save the path only
             if "Variable" in node.get("level", "") or not node.get("children"):
-                definition = node.get("definition", "")
-                if definition:
-                    collected_paths.append(f"{node_path} (Definition: {definition})")
-                else:
-                    collected_paths.append(node_path)
+                collected_paths.append(node_path)
             
             # Dig deeper into children
             for child in node.get("children", []):
@@ -110,11 +107,11 @@ def classify_individual_topic(topic_name: str):
         response = llm.invoke(prompt.format(title=state["title"], abstract=state["abstract"], sub_tree=target_sub_tree))
         raw_keywords = [k.strip() for k in response.content.split(",") if k.strip()]
         
+        # Immediate validation pass inside the node branch
         valid_set = set()
         for line in target_sub_tree.split("\n"):
             if line.strip():
                 path = line.replace("- ", "").strip()
-                path = re.sub(r"\s*\(Definition:.*?\)$", "", path).strip()
                 valid_set.add(path)
                 
         valid_kws = [kw for kw in raw_keywords if kw in valid_set]
@@ -160,7 +157,7 @@ def run_agent_classifier(title, abstract):
         
     inputs = {"title": title, "abstract": abstract}
     
-    # Execute your exact notebook graph synchronously
+    # Execute graph synchronously
     output = app.invoke(inputs)
     
     chosen_topics = output.get("chosen_topics", [])
@@ -169,11 +166,10 @@ def run_agent_classifier(title, abstract):
     
     topics_str = ", ".join(chosen_topics)
     
-    # Requirement 1: Format output keywords to prepend their corresponding parent Topic
+    # Format output keywords to prepend their corresponding parent Topic
     formatted_keywords = []
     for kw in predicted_kws:
         matched_topic = "UNKNOWN"
-        # Find which topic sub-tree this validated keyword originally belonged to
         for topic in chosen_topics:
             sub_tree_text = SUB_TREE_LOOKUP.get(topic, "")
             if kw in sub_tree_text:
@@ -186,7 +182,7 @@ def run_agent_classifier(title, abstract):
     else:
         keywords_str = "\n".join(formatted_keywords)
         
-    # Requirement 2: Format separate display for caught and filtered hallucinations
+    # Format separate display for caught and filtered hallucinations
     if not invalid_kws:
         invalid_str = "None! The agent validation pass achieved 100% data integrity."
     else: