added logic for get_figures and grader implemntation

server/earnings_analyst_environment.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import math
 import os
+import json
 import random
 from typing import Any
 from uuid import uuid4
@@ -120,10 +121,16 @@ class EarningsAnalystEnvironment(Environment):
             Terminal observation with reward and metadata including ground truth.
         """
         self._state.step_count += 1
-        label_col = self._cfg["label_col"]
-        label_values = list(self._cfg["label_values"])
+        label_col = self._cfg.get("label_col", "symbol")
+        label_values = list(self._cfg.get("label_values", []))
         row = self._current_row or {}
-        ground_truth = str(row.get(label_col, "")).strip()
+
+        # Handle composite ground truth if multiple columns are specified (e.g. for get_figures)
+        if "xbrl_columns" in self._cfg:
+            gt_data = {col: row.get(col) for col in self._cfg["xbrl_columns"]}
+            ground_truth = json.dumps(gt_data)
+        else:
+            ground_truth = str(row.get(label_col, "")).strip()
 
         grade_fn = get_grader(self._task_id)
         reward = float(

tasks/get_figures/grader.py

@@ -2,13 +2,78 @@
 
 from __future__ import annotations
 import json
+import math
+from typing import Any
+
+
+def _safe_float(val: Any) -> float | None:
+    if val is None or val == "":
+        return None
+    try:
+        return float(val)
+    except (ValueError, TypeError):
+        return None
+
+
+def _get_score(pred: float | None, target: float | None, tolerance: float = 0.01) -> float:
+    """Compare pred to target with relative error tolerance."""
+    if target is None:
+        # If ground truth is null, reward 1.0 if prediction is also null, else 0.0
+        return 1.0 if pred is None else 0.0
+    
+    if pred is None:
+        return 0.0
+    
+    if abs(target) < 1e-9:
+        return 1.0 if abs(pred) < 1e-9 else 0.0
+    
+    relative_error = abs(pred - target) / abs(target)
+    return 1.0 if relative_error <= tolerance else 0.0
+
+
+def _flatten_metrics(data: dict[str, Any]) -> dict[str, float | None]:
+    """Helper to flatten the nested metrics JSON provided by the agent."""
+    flat = {}
+    for section in ["income_statement", "balance_sheet", "cash_flow"]:
+        if section in data and isinstance(data[section], dict):
+            for key, val in data[section].items():
+                flat[key] = _safe_float(val)
+    return flat
 
 
 def grade(predicted: str, ground_truth: str, label_values: list[str]) -> float:
     """
-    Score the agent's extraction performance.
+    Score the agent's extraction performance across multiple financial metrics.
+    
+    Args:
+        predicted: Agent's response string (expected JSON).
+        ground_truth: Environment's packed JSON string of XBRL values.
+        label_values: Unused.
     
-    Currently a stub until XBRL ground truth is provided.
-    Always returns 0.0 with implemented: False in spec.
+    Returns:
+        Average score (0.0 to 1.0) across all Metrics.
     """
-    return 0.0
+    try:
+        pred_data = json.loads(predicted)
+        target_data = json.loads(ground_truth)
+    except (json.JSONDecodeError, TypeError):
+        return 0.0
+
+    # Flatten the agent's nested response
+    pred_metrics = _flatten_metrics(pred_data)
+    
+    # Environment's target_data is already flat (mapping column_name -> value)
+    # We need to map the canonical keys (revenue, etc.) to the column values.
+    from .spec import METRIC_TO_COLUMN
+    
+    scores = []
+    for metric_key, col_name in METRIC_TO_COLUMN.items():
+        pred_val = pred_metrics.get(metric_key)
+        target_val = _safe_float(target_data.get(col_name))
+        
+        scores.append(_get_score(pred_val, target_val))
+        
+    if not scores:
+        return 0.0
+        
+    return sum(scores) / len(scores)

tasks/get_figures/spec.py

@@ -5,25 +5,65 @@ from ..types import TaskSpec
 
 CANONICAL_TASK_ID = "get_figures"
 
+# Mapping from JSON metric keys to dataset XBRL columns
+METRIC_TO_COLUMN: dict[str, str] = {
+    "revenue": "xbrl_revenue",
+    "cost_of_revenue": "xbrl_cost_of_revenue",
+    "gross_profit": "xbrl_gross_profit",
+    "operating_income": "xbrl_operating_income",
+    "net_income": "xbrl_net_income",
+    "eps_basic": "xbrl_eps_basic",
+    "eps_diluted": "xbrl_eps_diluted",
+    "cash_and_cash_equivalents": "xbrl_cash_and_cash_equivalents",
+    "total_assets": "xbrl_total_assets",
+    "total_liabilities": "xbrl_total_liabilities",
+    "net_cash_operating_activities": "xbrl_net_cash_operating_activities",
+    "capital_expenditures": "xbrl_capital_expenditures",
+}
+
 SPEC: TaskSpec = {
     "task_id": CANONICAL_TASK_ID,
-    "implemented": False,  # Safety gate: set to True once ground truth column is confirmed.
+    "implemented": True,
     "text_cols": [
         "earnings_transcript",
         "press_release_8k_body",
         "press_release_ex991",
         "press_release_ex992",
-        "press_release_sources",
     ],
-    "numerical_cols": [],
-    "label_col": "symbol",  # Placeholder
+    "numerical_cols": [
+        "price_momentum_30d",
+        "avg_volume_20d",
+    ],
+    "label_col": "xbrl_revenue",  # Primary ground truth column
     "label_values": [],
     "task_instruction": (
         "Extract key financial figures from the provided earnings call materials.\n\n"
-        "Return a JSON object matching this exact schema:\n"
-        '{"revenue": <float>, "net_income": <float>, "eps": <float>}\n\n'
-        "Use the currency specified in the documents. If a figure is not found, use null.\n"
-        "Do not include any other keys or explanation."
+        "Return a JSON object matching this exact US-GAAP taxonomy schema:\n"
+        "{\n"
+        '  "taxonomy_version": "us-gaap-2024",\n'
+        '  "income_statement": {\n'
+        '    "revenue": <float>,\n'
+        '    "cost_of_revenue": <float>,\n'
+        '    "gross_profit": <float>,\n'
+        '    "operating_income": <float>,\n'
+        '    "net_income": <float>,\n'
+        '    "eps_basic": <float>,\n'
+        '    "eps_diluted": <float>\n'
+        "  },\n"
+        '  "balance_sheet": {\n'
+        '    "cash_and_cash_equivalents": <float>,\n'
+        '    "total_assets": <float>,\n'
+        '    "total_liabilities": <float>\n'
+        "  },\n"
+        '  "cash_flow": {\n'
+        '    "net_cash_operating_activities": <float>,\n'
+        '    "capital_expenditures": <float>\n'
+        "  }\n"
+        "}\n\n"
+        "Values should ideally be in USD. If a figure is not found or not mentioned, use null.\n"
+        "Do not include any other keys, explanations, or markdown blocks."
     ),
-    "kind": "other",
+    "kind": "extraction",
+    # Metadata for the environment to pack all ground truth figures
+    "xbrl_columns": list(METRIC_TO_COLUMN.values()),
 }