File size: 1,417 Bytes
1c03487 0092607 1c03487 0092607 1c03487 0092607 1c03487 0092607 1c03487 0092607 1c03487 9eafd01 1c03487 0092607 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from typing import List
from core.trajectory import EpisodicMemory
def compute_advantage(
current_reward: float,
completed_rewards: List[float],
) -> float:
"""
GRPO advantage = R_i - mean(R_completed).
Positive means this rollout was better than average; negative means worse.
Returns 0.0 on the first rollout where there's nothing to compare against.
"""
if not completed_rewards:
return 0.0
mean = sum(completed_rewards) / len(completed_rewards)
return round(current_reward - mean, 4)
def should_reinforce(advantage: float) -> bool:
# Small negative margin is allowed to encourage exploration on borderline rollouts.
return advantage > -0.5
def update_memory(
memory: EpisodicMemory,
trajectory: List[dict],
advantage: float,
) -> int:
"""
Store positive-reward steps from this trajectory into episodic memory
if the rollout was at or above average (advantage > threshold).
Returns the number of steps stored. Bad rollouts leave memory unchanged.
"""
if not should_reinforce(advantage):
return 0
stored = 0
for step_data in trajectory:
if step_data["reward"] > -0.3:
memory.store(step_data["obs"], step_data["action"], step_data["reward"])
stored += 1
return stored
|