DI-engine/ding/hpc_rl/tests/test_ppo.py

import time
import torch
import torch.nn.functional as F
from hpc_rll.origin.ppo import ppo_error, ppo_data
from hpc_rll.rl_utils.ppo import PPO
from testbase import mean_relative_error, times

assert torch.cuda.is_available()
use_cuda = True

B = 128
N = 128
clip_ratio = 0.2
use_value_clip = True
dual_clip = None


def ppo_val():
    ori_logits_new = torch.randn(B, N)
    ori_logits_old = torch.randn(B, N)
    ori_action = torch.randint(0, N, size=(B, ))
    ori_value_new = torch.randn(B)
    ori_value_old = torch.randn(B)
    ori_adv = torch.randn(B)
    ori_return = torch.randn(B)
    ori_weight = torch.randn(B)

    hpc_logits_new = ori_logits_new.clone().detach()
    hpc_logits_old = ori_logits_old.clone().detach()
    hpc_action = ori_action.clone().detach()
    hpc_value_new = ori_value_new.clone().detach()
    hpc_value_old = ori_value_old.clone().detach()
    hpc_adv = ori_adv.clone().detach()
    hpc_return = ori_return.clone().detach()
    hpc_weight = ori_weight.clone().detach()
    hpc_ppo = PPO(B, N)

    if use_cuda:
        ori_logits_new = ori_logits_new.cuda()
        ori_logits_old = ori_logits_old.cuda()
        ori_action = ori_action.cuda()
        ori_value_new = ori_value_new.cuda()
        ori_value_old = ori_value_old.cuda()
        ori_adv = ori_adv.cuda()
        ori_return = ori_return.cuda()
        ori_weight = ori_weight.cuda()

        hpc_logits_new = hpc_logits_new.cuda()
        hpc_logits_old = hpc_logits_old.cuda()
        hpc_action = hpc_action.cuda()
        hpc_value_new = hpc_value_new.cuda()
        hpc_value_old = hpc_value_old.cuda()
        hpc_adv = hpc_adv.cuda()
        hpc_return = hpc_return.cuda()
        hpc_weight = hpc_weight.cuda()
        hpc_ppo = hpc_ppo.cuda()

    ori_logits_new.requires_grad_(True)
    ori_value_new.requires_grad_(True)
    ori_loss, ori_info = ppo_error(
        ppo_data(
            ori_logits_new, ori_logits_old, ori_action, ori_value_new, ori_value_old, ori_adv, ori_return, ori_weight
        ), clip_ratio, use_value_clip, dual_clip
    )
    ori_loss = sum(ori_loss)
    ori_loss.backward()

    hpc_logits_new.requires_grad_(True)
    hpc_value_new.requires_grad_(True)
    hpc_loss, hpc_info = hpc_ppo(
        hpc_logits_new, hpc_logits_old, hpc_action, hpc_value_new, hpc_value_old, hpc_adv, hpc_return, hpc_weight,
        clip_ratio, use_value_clip, dual_clip
    )
    hpc_loss = sum(hpc_loss)
    hpc_loss.backward()

    print("ori_info: " + str(ori_info))
    print("hpc_info: " + str(hpc_info))
    mre = mean_relative_error(
        torch.flatten(ori_loss).cpu().detach().numpy(),
        torch.flatten(hpc_loss).cpu().detach().numpy()
    )
    print("ppo fp loss mean_relative_error: " + str(mre))
    mre = mean_relative_error(
        torch.flatten(ori_logits_new.grad).cpu().detach().numpy(),
        torch.flatten(hpc_logits_new.grad).cpu().detach().numpy()
    )
    print("ppo bp logits_new mean_relative_error: " + str(mre))
    mre = mean_relative_error(
        torch.flatten(ori_value_new.grad).cpu().detach().numpy(),
        torch.flatten(hpc_value_new.grad).cpu().detach().numpy()
    )
    print("ppo bp value_new mean_relative_error: " + str(mre))


def ppo_perf():
    ori_logits_new = torch.randn(B, N)
    ori_logits_old = torch.randn(B, N)
    ori_action = torch.randint(0, N, size=(B, ))
    ori_value_new = torch.randn(B)
    ori_value_old = torch.randn(B)
    ori_adv = torch.randn(B)
    ori_return = torch.randn(B)
    ori_weight = torch.randn(B)

    hpc_logits_new = ori_logits_new.clone().detach()
    hpc_logits_old = ori_logits_old.clone().detach()
    hpc_action = ori_action.clone().detach()
    hpc_value_new = ori_value_new.clone().detach()
    hpc_value_old = ori_value_old.clone().detach()
    hpc_adv = ori_adv.clone().detach()
    hpc_return = ori_return.clone().detach()
    hpc_weight = ori_weight.clone().detach()
    hpc_ppo = PPO(B, N)

    if use_cuda:
        ori_logits_new = ori_logits_new.cuda()
        ori_logits_old = ori_logits_old.cuda()
        ori_action = ori_action.cuda()
        ori_value_new = ori_value_new.cuda()
        ori_value_old = ori_value_old.cuda()
        ori_adv = ori_adv.cuda()
        ori_return = ori_return.cuda()
        ori_weight = ori_weight.cuda()

        hpc_logits_new = hpc_logits_new.cuda()
        hpc_logits_old = hpc_logits_old.cuda()
        hpc_action = hpc_action.cuda()
        hpc_value_new = hpc_value_new.cuda()
        hpc_value_old = hpc_value_old.cuda()
        hpc_adv = hpc_adv.cuda()
        hpc_return = hpc_return.cuda()
        hpc_weight = hpc_weight.cuda()
        hpc_ppo = hpc_ppo.cuda()

    ori_logits_new.requires_grad_(True)
    ori_value_new.requires_grad_(True)
    for i in range(times):
        t = time.time()
        ori_loss, ori_info = ppo_error(
            ppo_data(
                ori_logits_new, ori_logits_old, ori_action, ori_value_new, ori_value_old, ori_adv, ori_return,
                ori_weight
            ), clip_ratio, use_value_clip, dual_clip
        )
        ori_loss = sum(ori_loss)
        ori_loss.backward()
        if use_cuda:
            torch.cuda.synchronize()
        print('epoch: {}, origin ppo cost time: {}'.format(i, time.time() - t))

    hpc_logits_new.requires_grad_(True)
    hpc_value_new.requires_grad_(True)
    for i in range(times):
        t = time.time()
        hpc_loss, hpc_info = hpc_ppo(
            hpc_logits_new, hpc_logits_old, hpc_action, hpc_value_new, hpc_value_old, hpc_adv, hpc_return, hpc_weight,
            clip_ratio, use_value_clip, dual_clip
        )
        hpc_loss = sum(hpc_loss)
        hpc_loss.backward()
        if use_cuda:
            torch.cuda.synchronize()
        print('epoch: {}, hpc ppo cost time: {}'.format(i, time.time() - t))


if __name__ == '__main__':
    print(
        "target problem: B = {}, N = {}, clip_ratio = {}, use_value_clip = {}, dual_clip = {}".format(
            B, N, clip_ratio, use_value_clip, dual_clip
        )
    )
    print("================run ppo validation test================")
    ppo_val()
    print("================run ppo performance test================")
    ppo_perf()