Instructions to use cs552-the-expendables/kto_base_base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use cs552-the-expendables/kto_base_base with PEFT:

from peft import PeftModel
from transformers import AutoModelForCausalLM

base_model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3.5-4B")
model = PeftModel.from_pretrained(base_model, "cs552-the-expendables/kto_base_base")

Transformers

How to use cs552-the-expendables/kto_base_base with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="cs552-the-expendables/kto_base_base")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# Load model directly
from transformers import AutoModel
model = AutoModel.from_pretrained("cs552-the-expendables/kto_base_base", dtype="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use cs552-the-expendables/kto_base_base with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "cs552-the-expendables/kto_base_base"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "cs552-the-expendables/kto_base_base",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/cs552-the-expendables/kto_base_base

SGLang

How to use cs552-the-expendables/kto_base_base with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "cs552-the-expendables/kto_base_base" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "cs552-the-expendables/kto_base_base",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "cs552-the-expendables/kto_base_base" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "cs552-the-expendables/kto_base_base",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use cs552-the-expendables/kto_base_base with Docker Model Runner:
```
docker model run hf.co/cs552-the-expendables/kto_base_base
```

kto_base_base / trainer_state.json

dimgor

Upload trainer_state.json with huggingface_hub

b75820b verified 29 days ago

Raw

History Blame Contribute Delete

86.8 kB

	{
	"best_global_step": null,
	"best_metric": null,
	"best_model_checkpoint": null,
	"epoch": 2.0,
	"eval_steps": 50,
	"global_step": 1334,
	"is_hyper_param_search": false,
	"is_local_process_zero": true,
	"is_world_process_zero": true,
	"log_history": [
	{
	"epoch": 0.015012197410395946,
	"grad_norm": 0.48664575815200806,
	"kl": 0.05595267936587334,
	"learning_rate": 2.686567164179105e-06,
	"logits/chosen": -105151202.07792208,
	"logits/rejected": -100224814.26506025,
	"logps/chosen": -32.888798701298704,
	"logps/rejected": -30.840290850903614,
	"loss": 0.5002947807312011,
	"rewards/chosen": -0.0017914044392573368,
	"rewards/margins": -0.003978508190451383,
	"rewards/rejected": 0.0021871037511940463,
	"step": 10
	},
	{
	"epoch": 0.030024394820791892,
	"grad_norm": 0.7029150128364563,
	"kl": 0.08854874223470688,
	"learning_rate": 5.671641791044776e-06,
	"logits/chosen": -90880536.38095239,
	"logits/rejected": -110196500.21052632,
	"logps/chosen": -39.982947939918155,
	"logps/rejected": -31.82884457236842,
	"loss": 0.4987481594085693,
	"rewards/chosen": 0.007288790174892971,
	"rewards/margins": 0.008891924347420385,
	"rewards/rejected": -0.0016031341725274136,
	"step": 20
	},
	{
	"epoch": 0.04503659223118784,
	"grad_norm": 0.7273629903793335,
	"kl": 0.26002588868141174,
	"learning_rate": 8.656716417910447e-06,
	"logits/chosen": -98148453.05263157,
	"logits/rejected": -101435257.90476191,
	"logps/chosen": -34.69019197162829,
	"logps/rejected": -27.827526274181547,
	"loss": 0.49477639198303225,
	"rewards/chosen": 0.038317783882743435,
	"rewards/margins": 0.04702380150183102,
	"rewards/rejected": -0.008706017619087583,
	"step": 30
	},
	{
	"epoch": 0.060048789641583784,
	"grad_norm": 0.8483396172523499,
	"kl": 0.9080759286880493,
	"learning_rate": 1.1641791044776121e-05,
	"logits/chosen": -79740646.0289855,
	"logits/rejected": -91644567.91208792,
	"logps/chosen": -36.05313773777174,
	"logps/rejected": -33.51322115384615,
	"loss": 0.4763810157775879,
	"rewards/chosen": 0.14765035933342532,
	"rewards/margins": 0.1901713592418726,
	"rewards/rejected": -0.042520999908447266,
	"step": 40
	},
	{
	"epoch": 0.07506098705197974,
	"grad_norm": 1.2962456941604614,
	"kl": 2.3752338886260986,
	"learning_rate": 1.4626865671641792e-05,
	"logits/chosen": -82581724.27906977,
	"logits/rejected": -105110202.8108108,
	"logps/chosen": -26.611898732739824,
	"logps/rejected": -32.0681812183277,
	"loss": 0.4478025436401367,
	"rewards/chosen": 0.3435138879820358,
	"rewards/margins": 0.5002580459278234,
	"rewards/rejected": -0.15674415794578758,
	"step": 50
	},
	{
	"epoch": 0.07506098705197974,
	"eval_kl": 1.266696572303772,
	"eval_logits/chosen": -55813307.64397906,
	"eval_logits/rejected": -56554646.11518325,
	"eval_logps/chosen": -31.56733679646597,
	"eval_logps/rejected": -35.57937765870419,
	"eval_loss": 0.3994504511356354,
	"eval_rewards/chosen": 0.5163626146566181,
	"eval_rewards/margins": 0.8252790161452368,
	"eval_rewards/rejected": -0.3089164014886187,
	"eval_runtime": 464.3345,
	"eval_samples_per_second": 1.645,
	"eval_steps_per_second": 0.823,
	"step": 50
	},
	{
	"epoch": 0.09007318446237568,
	"grad_norm": 0.775963306427002,
	"kl": 4.466587066650391,
	"learning_rate": 1.7611940298507464e-05,
	"logits/chosen": -83022252.65116279,
	"logits/rejected": -83638320.43243243,
	"logps/chosen": -29.599955714026162,
	"logps/rejected": -38.7579246727196,
	"loss": 0.3778259754180908,
	"rewards/chosen": 0.6707564509192179,
	"rewards/margins": 1.2162449115630893,
	"rewards/rejected": -0.5454884606438715,
	"step": 60
	},
	{
	"epoch": 0.10508538187277162,
	"grad_norm": 0.43517357110977173,
	"kl": 5.635891914367676,
	"learning_rate": 1.9999877036668485e-05,
	"logits/chosen": -86946379.63636364,
	"logits/rejected": -74469447.1111111,
	"logps/chosen": -25.857749245383523,
	"logps/rejected": -35.837982177734375,
	"loss": 0.3408613443374634,
	"rewards/chosen": 0.8303727236661044,
	"rewards/margins": 1.9862453865282461,
	"rewards/rejected": -1.1558726628621419,
	"step": 70
	},
	{
	"epoch": 0.12009757928316757,
	"grad_norm": 0.37438932061195374,
	"kl": 5.818446636199951,
	"learning_rate": 1.999557363757665e-05,
	"logits/chosen": -97060782.08,
	"logits/rejected": -77247716.89411765,
	"logps/chosen": -30.320673828125,
	"logps/rejected": -56.52916475183824,
	"loss": 0.2436617136001587,
	"rewards/chosen": 1.0769728597005208,
	"rewards/margins": 3.3871797329771756,
	"rewards/rejected": -2.3102068732766545,
	"step": 80
	},
	{
	"epoch": 0.13510977669356353,
	"grad_norm": 0.4023344814777374,
	"kl": 5.037129878997803,
	"learning_rate": 1.9985125095570854e-05,
	"logits/chosen": -88758926.58227848,
	"logits/rejected": -90645194.27160494,
	"logps/chosen": -28.795388523536392,
	"logps/rejected": -63.26287012924383,
	"loss": 0.2622586011886597,
	"rewards/chosen": 0.9623794555664062,
	"rewards/margins": 4.024350295832128,
	"rewards/rejected": -3.0619708402657215,
	"step": 90
	},
	{
	"epoch": 0.15012197410395947,
	"grad_norm": 0.3208625614643097,
	"kl": 4.7330522537231445,
	"learning_rate": 1.996853783427282e-05,
	"logits/chosen": -78684659.8095238,
	"logits/rejected": -79006693.05263157,
	"logps/chosen": -23.673159644717263,
	"logps/rejected": -65.49229029605263,
	"loss": 0.27015070915222167,
	"rewards/chosen": 0.9256639026460194,
	"rewards/margins": 4.187760106901775,
	"rewards/rejected": -3.2620962042557564,
	"step": 100
	},
	{
	"epoch": 0.15012197410395947,
	"eval_kl": 0.14395534992218018,
	"eval_logits/chosen": -47605652.77486911,
	"eval_logits/rejected": -38367379.43455497,
	"eval_logps/chosen": -27.383620337041886,
	"eval_logps/rejected": -67.5675719895288,
	"eval_loss": 0.20005758106708527,
	"eval_rewards/chosen": 0.9347344043991328,
	"eval_rewards/margins": 4.442470530564872,
	"eval_rewards/rejected": -3.5077361261657396,
	"eval_runtime": 472.543,
	"eval_samples_per_second": 1.617,
	"eval_steps_per_second": 0.808,
	"step": 100
	},
	{
	"epoch": 0.16513417151435542,
	"grad_norm": 0.44810113310813904,
	"kl": 4.619333267211914,
	"learning_rate": 1.994582205130551e-05,
	"logits/chosen": -76966491.1780822,
	"logits/rejected": -71586910.16091955,
	"logps/chosen": -26.702449432791095,
	"logps/rejected": -66.4336218121408,
	"loss": 0.2479196548461914,
	"rewards/chosen": 0.9265415766467787,
	"rewards/margins": 4.2691862790758375,
	"rewards/rejected": -3.342644702429059,
	"step": 110
	},
	{
	"epoch": 0.18014636892475136,
	"grad_norm": 0.34525591135025024,
	"kl": 5.025256156921387,
	"learning_rate": 1.991699171202376e-05,
	"logits/chosen": -84793181.36470588,
	"logits/rejected": -70789720.74666667,
	"logps/chosen": -27.854406020220587,
	"logps/rejected": -62.20436197916667,
	"loss": 0.2833747148513794,
	"rewards/chosen": 0.8931190939510569,
	"rewards/margins": 4.17925760306564,
	"rewards/rejected": -3.286138509114583,
	"step": 120
	},
	{
	"epoch": 0.1951585663351473,
	"grad_norm": 0.38824889063835144,
	"kl": 5.002859592437744,
	"learning_rate": 1.9882064540928598e-05,
	"logits/chosen": -80103908.47311828,
	"logits/rejected": -67727787.94029851,
	"logps/chosen": -24.253184328797044,
	"logps/rejected": -76.31905900186567,
	"loss": 0.2879242658615112,
	"rewards/chosen": 0.9358312135101646,
	"rewards/margins": 4.883322759685538,
	"rewards/rejected": -3.9474915461753732,
	"step": 130
	},
	{
	"epoch": 0.21017076374554325,
	"grad_norm": 0.4769190847873688,
	"kl": 4.515671730041504,
	"learning_rate": 1.9841062010770435e-05,
	"logits/chosen": -79682590.47619048,
	"logits/rejected": -53528855.578947365,
	"logps/chosen": -31.625252859933035,
	"logps/rejected": -65.04932925575658,
	"loss": 0.268853759765625,
	"rewards/chosen": 0.9304619743710473,
	"rewards/margins": 4.450175526745636,
	"rewards/rejected": -3.5197135523745886,
	"step": 140
	},
	{
	"epoch": 0.2251829611559392,
	"grad_norm": 0.6323740482330322,
	"kl": 4.37551736831665,
	"learning_rate": 1.9794009329347926e-05,
	"logits/chosen": -86433298.73170732,
	"logits/rejected": -85731268.92307693,
	"logps/chosen": -33.249347965891765,
	"logps/rejected": -68.0100348056891,
	"loss": 0.25741305351257326,
	"rewards/chosen": 0.9401206970214844,
	"rewards/margins": 4.5076380998660355,
	"rewards/rejected": -3.567517402844551,
	"step": 150
	},
	{
	"epoch": 0.2251829611559392,
	"eval_kl": 0.17389115691184998,
	"eval_logits/chosen": -45713702.86910995,
	"eval_logits/rejected": -34640582.366492145,
	"eval_logps/chosen": -26.543809820844242,
	"eval_logps/rejected": -71.86549513252618,
	"eval_loss": 0.1908683031797409,
	"eval_rewards/chosen": 1.0187157436191099,
	"eval_rewards/margins": 4.956243544973004,
	"eval_rewards/rejected": -3.937527801353894,
	"eval_runtime": 464.1447,
	"eval_samples_per_second": 1.646,
	"eval_steps_per_second": 0.823,
	"step": 150
	},
	{
	"epoch": 0.24019515856633514,
	"grad_norm": 0.3375397026538849,
	"kl": 4.020479202270508,
	"learning_rate": 1.974093542401054e-05,
	"logits/chosen": -62679096.88888889,
	"logits/rejected": -77540241.82278481,
	"logps/chosen": -22.955916039737655,
	"logps/rejected": -71.9782127670095,
	"loss": 0.26079225540161133,
	"rewards/chosen": 0.8830830138406636,
	"rewards/margins": 4.736599025139121,
	"rewards/rejected": -3.8535160112984572,
	"step": 160
	},
	{
	"epoch": 0.2552073559767311,
	"grad_norm": 0.4141367971897125,
	"kl": 3.990988254547119,
	"learning_rate": 1.968187292387444e-05,
	"logits/chosen": -84086227.2,
	"logits/rejected": -68478252.8,
	"logps/chosen": -29.576809692382813,
	"logps/rejected": -76.20518798828125,
	"loss": 0.22698066234588624,
	"rewards/chosen": 1.1348169326782227,
	"rewards/margins": 5.321233177185058,
	"rewards/rejected": -4.186416244506836,
	"step": 170
	},
	{
	"epoch": 0.27021955338712705,
	"grad_norm": 0.2972460687160492,
	"kl": 3.929959774017334,
	"learning_rate": 1.9616858139762534e-05,
	"logits/chosen": -99423652.8219178,
	"logits/rejected": -75411597.2413793,
	"logps/chosen": -23.80077288901969,
	"logps/rejected": -63.870745779454026,
	"loss": 0.23734564781188966,
	"rewards/chosen": 0.9383544921875,
	"rewards/margins": 4.621817051679239,
	"rewards/rejected": -3.6834625594917383,
	"step": 180
	},
	{
	"epoch": 0.285231750797523,
	"grad_norm": 0.41057029366493225,
	"kl": 4.216883659362793,
	"learning_rate": 1.954593104188107e-05,
	"logits/chosen": -75821622.61333333,
	"logits/rejected": -63695125.08235294,
	"logps/chosen": -24.083863932291667,
	"logps/rejected": -68.76608455882354,
	"loss": 0.2280487060546875,
	"rewards/chosen": 1.1367711385091146,
	"rewards/margins": 5.0540437286975335,
	"rewards/rejected": -3.917272590188419,
	"step": 190
	},
	{
	"epoch": 0.30024394820791894,
	"grad_norm": 0.32265180349349976,
	"kl": 4.959702491760254,
	"learning_rate": 1.9469135235246503e-05,
	"logits/chosen": -70969464.80898876,
	"logits/rejected": -63905417.01408451,
	"logps/chosen": -26.195647164676966,
	"logps/rejected": -72.58189370598592,
	"loss": 0.2571362018585205,
	"rewards/chosen": 1.1260040797544328,
	"rewards/margins": 5.329004430491668,
	"rewards/rejected": -4.203000350737236,
	"step": 200
	},
	{
	"epoch": 0.30024394820791894,
	"eval_kl": 0.15900468826293945,
	"eval_logits/chosen": -43982228.77486911,
	"eval_logits/rejected": -31387007.329842933,
	"eval_logps/chosen": -26.374332767506544,
	"eval_logps/rejected": -77.79594445353403,
	"eval_loss": 0.18420633673667908,
	"eval_rewards/chosen": 1.0356630654859293,
	"eval_rewards/margins": 5.566235287651342,
	"eval_rewards/rejected": -4.5305722221654126,
	"eval_runtime": 418.7669,
	"eval_samples_per_second": 1.824,
	"eval_steps_per_second": 0.912,
	"step": 200
	},
	{
	"epoch": 0.3152561456183149,
	"grad_norm": 0.3199129104614258,
	"kl": 3.8654441833496094,
	"learning_rate": 1.9386517932877725e-05,
	"logits/chosen": -63076752.69565217,
	"logits/rejected": -68469287.38461539,
	"logps/chosen": -26.9500927026721,
	"logps/rejected": -77.23451450892857,
	"loss": 0.23405146598815918,
	"rewards/chosen": 0.9223934671153193,
	"rewards/margins": 5.440813340698767,
	"rewards/rejected": -4.518419873583448,
	"step": 210
	},
	{
	"epoch": 0.33026834302871083,
	"grad_norm": 0.4808659851551056,
	"kl": 4.875772476196289,
	"learning_rate": 1.9298129926770132e-05,
	"logits/chosen": -67988052.3956044,
	"logits/rejected": -63445236.86956522,
	"logps/chosen": -26.67523931146978,
	"logps/rejected": -78.15327785326087,
	"loss": 0.2701714277267456,
	"rewards/chosen": 1.0167858417217548,
	"rewards/margins": 5.61166012699123,
	"rewards/rejected": -4.594874285269475,
	"step": 220
	},
	{
	"epoch": 0.3452805404391068,
	"grad_norm": 0.2898392081260681,
	"kl": 3.1590735912323,
	"learning_rate": 1.920402555666941e-05,
	"logits/chosen": -67118973.96825397,
	"logits/rejected": -70008605.03092784,
	"logps/chosen": -28.408491830977184,
	"logps/rejected": -76.45369382248711,
	"loss": 0.19473642110824585,
	"rewards/chosen": 1.0573667495969743,
	"rewards/margins": 5.643574188586987,
	"rewards/rejected": -4.586207438990013,
	"step": 230
	},
	{
	"epoch": 0.3602927378495027,
	"grad_norm": 0.36392441391944885,
	"kl": 4.529810905456543,
	"learning_rate": 1.910426267666418e-05,
	"logits/chosen": -78076474.85057472,
	"logits/rejected": -74812261.69863014,
	"logps/chosen": -22.985556416127874,
	"logps/rejected": -74.91467786815069,
	"loss": 0.2578288555145264,
	"rewards/chosen": 1.0158540572243175,
	"rewards/margins": 5.46974408692252,
	"rewards/rejected": -4.4538900296982025,
	"step": 240
	},
	{
	"epoch": 0.37530493525989866,
	"grad_norm": 0.3606679439544678,
	"kl": 3.946249485015869,
	"learning_rate": 1.8998902619618117e-05,
	"logits/chosen": -75268391.38461539,
	"logits/rejected": -61223561.36585366,
	"logps/chosen": -24.976700220352566,
	"logps/rejected": -75.95566763528963,
	"loss": 0.23691632747650146,
	"rewards/chosen": 0.9890132806239984,
	"rewards/margins": 5.471519265046636,
	"rewards/rejected": -4.482505984422637,
	"step": 250
	},
	{
	"epoch": 0.37530493525989866,
	"eval_kl": 0.16116125881671906,
	"eval_logits/chosen": -45084184.12565445,
	"eval_logits/rejected": -31980281.298429318,
	"eval_logps/chosen": -26.101974087859947,
	"eval_logps/rejected": -80.09241553501309,
	"eval_loss": 0.18029513955116272,
	"eval_rewards/chosen": 1.0628989654061682,
	"eval_rewards/margins": 5.823118679186436,
	"eval_rewards/rejected": -4.760219713780268,
	"eval_runtime": 425.436,
	"eval_samples_per_second": 1.796,
	"eval_steps_per_second": 0.898,
	"step": 250
	},
	{
	"epoch": 0.3903171326702946,
	"grad_norm": 0.4404217302799225,
	"kl": 5.508103370666504,
	"learning_rate": 1.8888010159463294e-05,
	"logits/chosen": -64772149.89473684,
	"logits/rejected": -65391466.33846154,
	"logps/chosen": -25.71417814555921,
	"logps/rejected": -63.81729266826923,
	"loss": 0.28806486129760744,
	"rewards/chosen": 1.0554629677220395,
	"rewards/margins": 4.9188026706216785,
	"rewards/rejected": -3.8633397028996392,
	"step": 260
	},
	{
	"epoch": 0.40532933008069055,
	"grad_norm": 0.6277490258216858,
	"kl": 4.712449073791504,
	"learning_rate": 1.877165347137809e-05,
	"logits/chosen": -90587110.07594937,
	"logits/rejected": -68866022.71604939,
	"logps/chosen": -26.297134592563292,
	"logps/rejected": -75.93940489969135,
	"loss": 0.24233634471893312,
	"rewards/chosen": 1.0065902516811709,
	"rewards/margins": 5.576781434889119,
	"rewards/rejected": -4.570191183207948,
	"step": 270
	},
	{
	"epoch": 0.4203415274910865,
	"grad_norm": 0.32237356901168823,
	"kl": 3.980494737625122,
	"learning_rate": 1.864990408987401e-05,
	"logits/chosen": -70893069.83783785,
	"logits/rejected": -69000977.86046511,
	"logps/chosen": -28.647619298986488,
	"logps/rejected": -82.09332985101744,
	"loss": 0.2172463655471802,
	"rewards/chosen": 1.1764121184477936,
	"rewards/margins": 6.09591881244307,
	"rewards/rejected": -4.9195066939952765,
	"step": 280
	},
	{
	"epoch": 0.43535372490148244,
	"grad_norm": 0.3504326641559601,
	"kl": 4.214869022369385,
	"learning_rate": 1.852283686481721e-05,
	"logits/chosen": -68802216.58536585,
	"logits/rejected": -67867707.07692307,
	"logps/chosen": -27.0248398199314,
	"logps/rejected": -81.77991661658653,
	"loss": 0.242197847366333,
	"rewards/chosen": 1.0362097577350895,
	"rewards/margins": 5.799550634387733,
	"rewards/rejected": -4.763340876652644,
	"step": 290
	},
	{
	"epoch": 0.4503659223118784,
	"grad_norm": 0.33721521496772766,
	"kl": 4.173134803771973,
	"learning_rate": 1.8390529915411853e-05,
	"logits/chosen": -65034005.590361446,
	"logits/rejected": -67509134.96103896,
	"logps/chosen": -24.494224456419428,
	"logps/rejected": -84.23314478490259,
	"loss": 0.22235071659088135,
	"rewards/chosen": 1.1591761945241905,
	"rewards/margins": 6.0879243872494335,
	"rewards/rejected": -4.9287481927252434,
	"step": 300
	},
	{
	"epoch": 0.4503659223118784,
	"eval_kl": 0.17067986726760864,
	"eval_logits/chosen": -46740898.17801047,
	"eval_logits/rejected": -33324983.62303665,
	"eval_logps/chosen": -25.849294932100786,
	"eval_logps/rejected": -80.77142813318063,
	"eval_loss": 0.17754967510700226,
	"eval_rewards/chosen": 1.0881668809820844,
	"eval_rewards/margins": 5.916288685424165,
	"eval_rewards/rejected": -4.828121804442081,
	"eval_runtime": 427.9477,
	"eval_samples_per_second": 1.785,
	"eval_steps_per_second": 0.893,
	"step": 300
	},
	{
	"epoch": 0.46537811972227433,
	"grad_norm": 0.4849989116191864,
	"kl": 4.441267967224121,
	"learning_rate": 1.8253064582173457e-05,
	"logits/chosen": -69129365.06329113,
	"logits/rejected": -72012464.98765433,
	"logps/chosen": -21.719686387460442,
	"logps/rejected": -72.88394579475309,
	"loss": 0.24629390239715576,
	"rewards/chosen": 0.9837675456759296,
	"rewards/margins": 5.237387834666092,
	"rewards/rejected": -4.253620288990162,
	"step": 310
	},
	{
	"epoch": 0.4803903171326703,
	"grad_norm": 0.4396725594997406,
	"kl": 4.207592487335205,
	"learning_rate": 1.8110525376921863e-05,
	"logits/chosen": -79998642.60465117,
	"logits/rejected": -74981417.51351352,
	"logps/chosen": -25.47288335755814,
	"logps/rejected": -80.42956213048987,
	"loss": 0.2393803358078003,
	"rewards/chosen": 1.1084918532260628,
	"rewards/margins": 5.974426070524416,
	"rewards/rejected": -4.865934217298353,
	"step": 320
	},
	{
	"epoch": 0.4954025145430662,
	"grad_norm": 0.5366594195365906,
	"kl": 3.9820334911346436,
	"learning_rate": 1.796299993082451e-05,
	"logits/chosen": -74307654.1369863,
	"logits/rejected": -68484590.34482759,
	"logps/chosen": -26.794069055008563,
	"logps/rejected": -78.43782552083333,
	"loss": 0.22449307441711425,
	"rewards/chosen": 1.1078702325690282,
	"rewards/margins": 5.943478353191334,
	"rewards/rejected": -4.835608120622306,
	"step": 330
	},
	{
	"epoch": 0.5104147119534622,
	"grad_norm": 0.4778093099594116,
	"kl": 4.551473140716553,
	"learning_rate": 1.7810578940521998e-05,
	"logits/chosen": -98528541.76744185,
	"logits/rejected": -74760185.08108108,
	"logps/chosen": -20.286565736282704,
	"logps/rejected": -77.08236776815879,
	"loss": 0.2551869869232178,
	"rewards/chosen": 1.1039025949877361,
	"rewards/margins": 5.889928183564594,
	"rewards/rejected": -4.786025588576858,
	"step": 340
	},
	{
	"epoch": 0.5254269093638582,
	"grad_norm": 0.4355256259441376,
	"kl": 4.321000099182129,
	"learning_rate": 1.7653356112369e-05,
	"logits/chosen": -100444391.61904761,
	"logits/rejected": -81269827.36842105,
	"logps/chosen": -25.645176478794642,
	"logps/rejected": -87.44900030838816,
	"loss": 0.24073328971862792,
	"rewards/chosen": 1.0453554789225261,
	"rewards/margins": 6.272355263693291,
	"rewards/rejected": -5.226999784770765,
	"step": 350
	},
	{
	"epoch": 0.5254269093638582,
	"eval_kl": 0.1573699414730072,
	"eval_logits/chosen": -47666427.979057595,
	"eval_logits/rejected": -33615502.07329843,
	"eval_logps/chosen": -25.851928071825917,
	"eval_logps/rejected": -83.59740571825917,
	"eval_loss": 0.17489400506019592,
	"eval_rewards/chosen": 1.0879038865653632,
	"eval_rewards/margins": 6.198621979558655,
	"eval_rewards/rejected": -5.110718092993292,
	"eval_runtime": 422.5798,
	"eval_samples_per_second": 1.808,
	"eval_steps_per_second": 0.904,
	"step": 350
	},
	{
	"epoch": 0.5404391067742541,
	"grad_norm": 0.6740466356277466,
	"kl": 4.984511375427246,
	"learning_rate": 1.749142810482488e-05,
	"logits/chosen": -96085843.34883721,
	"logits/rejected": -83732369.2972973,
	"logps/chosen": -30.92190338844477,
	"logps/rejected": -78.17299672719595,
	"loss": 0.25183792114257814,
	"rewards/chosen": 1.143318264983421,
	"rewards/margins": 5.906809921312601,
	"rewards/rejected": -4.7634916563291805,
	"step": 360
	},
	{
	"epoch": 0.55545130418465,
	"grad_norm": 0.42306992411613464,
	"kl": 4.017538547515869,
	"learning_rate": 1.7324894469029405e-05,
	"logits/chosen": -83550086.4,
	"logits/rejected": -69219065.6,
	"logps/chosen": -21.729214477539063,
	"logps/rejected": -78.86099243164062,
	"loss": 0.2265170097351074,
	"rewards/chosen": 1.150559902191162,
	"rewards/margins": 5.680189037322998,
	"rewards/rejected": -4.529629135131836,
	"step": 370
	},
	{
	"epoch": 0.570463501595046,
	"grad_norm": 0.7183542251586914,
	"kl": 4.302075386047363,
	"learning_rate": 1.715385758760002e-05,
	"logits/chosen": -94022080.87671232,
	"logits/rejected": -67760139.77011494,
	"logps/chosen": -28.821867642337327,
	"logps/rejected": -88.21133598239942,
	"loss": 0.22173850536346434,
	"rewards/chosen": 1.1270856465378853,
	"rewards/margins": 6.429633759078295,
	"rewards/rejected": -5.30254811254041,
	"step": 380
	},
	{
	"epoch": 0.5854756990054419,
	"grad_norm": 0.4068809449672699,
	"kl": 4.413468360900879,
	"learning_rate": 1.697842261168843e-05,
	"logits/chosen": -89074789.20930232,
	"logits/rejected": -74542716.54054055,
	"logps/chosen": -24.867312409156977,
	"logps/rejected": -70.55822423986487,
	"loss": 0.24221227169036866,
	"rewards/chosen": 1.105927312096884,
	"rewards/margins": 5.485582892359764,
	"rewards/rejected": -4.37965558026288,
	"step": 390
	},
	{
	"epoch": 0.6004878964158379,
	"grad_norm": 0.3094225525856018,
	"kl": 3.9395923614501953,
	"learning_rate": 1.6798697396335102e-05,
	"logits/chosen": -84558361.28395061,
	"logits/rejected": -80714233.51898734,
	"logps/chosen": -30.648084852430557,
	"logps/rejected": -83.06282140031645,
	"loss": 0.22035319805145265,
	"rewards/chosen": 1.1816176426263503,
	"rewards/margins": 6.228112524944389,
	"rewards/rejected": -5.046494882318038,
	"step": 400
	},
	{
	"epoch": 0.6004878964158379,
	"eval_kl": 0.1634005606174469,
	"eval_logits/chosen": -50731554.84816754,
	"eval_logits/rejected": -36390836.942408375,
	"eval_logps/chosen": -25.668382894306284,
	"eval_logps/rejected": -84.24621134653141,
	"eval_loss": 0.1733660250902176,
	"eval_rewards/chosen": 1.1062583723617474,
	"eval_rewards/margins": 6.281857475560373,
	"eval_rewards/rejected": -5.175599103198626,
	"eval_runtime": 427.5005,
	"eval_samples_per_second": 1.787,
	"eval_steps_per_second": 0.894,
	"step": 400
	},
	{
	"epoch": 0.6155000938262338,
	"grad_norm": 0.4934229254722595,
	"kl": 3.653895616531372,
	"learning_rate": 1.6614792434161418e-05,
	"logits/chosen": -90473977.17333333,
	"logits/rejected": -89471861.45882353,
	"logps/chosen": -22.688899739583334,
	"logps/rejected": -79.36219554227941,
	"loss": 0.21901493072509765,
	"rewards/chosen": 1.0243033854166668,
	"rewards/margins": 5.917456317976409,
	"rewards/rejected": -4.893152932559743,
	"step": 410
	},
	{
	"epoch": 0.6305122912366298,
	"grad_norm": 0.4996384382247925,
	"kl": 3.6179280281066895,
	"learning_rate": 1.6426820787440366e-05,
	"logits/chosen": -85964416.0,
	"logits/rejected": -84499870.47619048,
	"logps/chosen": -26.152443333675986,
	"logps/rejected": -78.37731352306548,
	"loss": 0.2094388484954834,
	"rewards/chosen": 1.1747771815249795,
	"rewards/margins": 6.022950777135098,
	"rewards/rejected": -4.848173595610119,
	"step": 420
	},
	{
	"epoch": 0.6455244886470257,
	"grad_norm": 0.6519464254379272,
	"kl": 3.751269817352295,
	"learning_rate": 1.6234898018587336e-05,
	"logits/chosen": -82877628.63157895,
	"logits/rejected": -67029967.23809524,
	"logps/chosen": -25.796669407894736,
	"logps/rejected": -80.34830729166667,
	"loss": 0.21852760314941405,
	"rewards/chosen": 0.9713470057437295,
	"rewards/margins": 5.867524025135471,
	"rewards/rejected": -4.896177019391741,
	"step": 430
	},
	{
	"epoch": 0.6605366860574217,
	"grad_norm": 0.5263959765434265,
	"kl": 4.094634056091309,
	"learning_rate": 1.6039142119113956e-05,
	"logits/chosen": -80433825.68421052,
	"logits/rejected": -77660050.28571428,
	"logps/chosen": -23.38715402703536,
	"logps/rejected": -83.7175525483631,
	"loss": 0.2311333656311035,
	"rewards/chosen": 1.0536013151469983,
	"rewards/margins": 6.101049609650346,
	"rewards/rejected": -5.047448294503348,
	"step": 440
	},
	{
	"epoch": 0.6755488834678176,
	"grad_norm": 0.58863765001297,
	"kl": 5.108607292175293,
	"learning_rate": 1.58396734370885e-05,
	"logits/chosen": -92037412.57142857,
	"logits/rejected": -84992687.15789473,
	"logps/chosen": -34.8051031203497,
	"logps/rejected": -72.51608115748355,
	"loss": 0.25075249671936034,
	"rewards/chosen": 1.194018046061198,
	"rewards/margins": 5.641956663968271,
	"rewards/rejected": -4.447938617907073,
	"step": 450
	},
	{
	"epoch": 0.6755488834678176,
	"eval_kl": 0.15371429920196533,
	"eval_logits/chosen": -53600052.27225131,
	"eval_logits/rejected": -38773534.827225134,
	"eval_logps/chosen": -25.656866103566752,
	"eval_logps/rejected": -85.39284399541884,
	"eval_loss": 0.17146842181682587,
	"eval_rewards/chosen": 1.1074100514357004,
	"eval_rewards/margins": 6.397672483434228,
	"eval_rewards/rejected": -5.290262431998528,
	"eval_runtime": 428.4403,
	"eval_samples_per_second": 1.783,
	"eval_steps_per_second": 0.892,
	"step": 450
	},
	{
	"epoch": 0.6905610808782136,
	"grad_norm": 0.49697065353393555,
	"kl": 3.1813266277313232,
	"learning_rate": 1.5636614603147513e-05,
	"logits/chosen": -114965474.31884058,
	"logits/rejected": -84445184.0,
	"logps/chosen": -22.76505710767663,
	"logps/rejected": -89.04121415693682,
	"loss": 0.20205256938934327,
	"rewards/chosen": 0.9410718834918478,
	"rewards/margins": 6.368589281864959,
	"rewards/rejected": -5.427517398373111,
	"step": 460
	},
	{
	"epoch": 0.7055732782886095,
	"grad_norm": 0.6644816994667053,
	"kl": 3.073093891143799,
	"learning_rate": 1.543009045510419e-05,
	"logits/chosen": -89053725.25714286,
	"logits/rejected": -82642397.86666666,
	"logps/chosen": -27.184453473772322,
	"logps/rejected": -74.28021918402777,
	"loss": 0.1959690809249878,
	"rewards/chosen": 1.1658863612583705,
	"rewards/margins": 5.783280702621218,
	"rewards/rejected": -4.617394341362847,
	"step": 470
	},
	{
	"epoch": 0.7205854756990054,
	"grad_norm": 0.3856127858161926,
	"kl": 3.390268325805664,
	"learning_rate": 1.5220227961199767e-05,
	"logits/chosen": -105331782.1369863,
	"logits/rejected": -82775881.56321838,
	"logps/chosen": -26.920950275577912,
	"logps/rejected": -86.48434132543103,
	"loss": 0.1951006054878235,
	"rewards/chosen": 1.159071830854024,
	"rewards/margins": 6.306170415960884,
	"rewards/rejected": -5.1470985851068605,
	"step": 480
	},
	{
	"epoch": 0.7355976731094014,
	"grad_norm": 0.5393089056015015,
	"kl": 3.25313138961792,
	"learning_rate": 1.5007156142045176e-05,
	"logits/chosen": -91164856.8888889,
	"logits/rejected": -86136715.63636364,
	"logps/chosen": -23.249182807074654,
	"logps/rejected": -84.9217529296875,
	"loss": 0.21610660552978517,
	"rewards/chosen": 0.9365202585856119,
	"rewards/margins": 6.319411826856209,
	"rewards/rejected": -5.382891568270597,
	"step": 490
	},
	{
	"epoch": 0.7506098705197973,
	"grad_norm": 0.7707648873329163,
	"kl": 3.6630606651306152,
	"learning_rate": 1.4791005991300923e-05,
	"logits/chosen": -110857683.75308642,
	"logits/rejected": -82150322.2278481,
	"logps/chosen": -24.525343906732253,
	"logps/rejected": -82.56521954113924,
	"loss": 0.22861866950988768,
	"rewards/chosen": 0.9733524087034626,
	"rewards/margins": 6.094431865660543,
	"rewards/rejected": -5.121079456957081,
	"step": 500
	},
	{
	"epoch": 0.7506098705197973,
	"eval_kl": 0.1316855102777481,
	"eval_logits/chosen": -54686553.801047124,
	"eval_logits/rejected": -39620532.942408375,
	"eval_logps/chosen": -25.837093013743456,
	"eval_logps/rejected": -88.067878763089,
	"eval_loss": 0.16978919506072998,
	"eval_rewards/chosen": 1.0893871047733965,
	"eval_rewards/margins": 6.647152566160831,
	"eval_rewards/rejected": -5.557765461387435,
	"eval_runtime": 424.0565,
	"eval_samples_per_second": 1.802,
	"eval_steps_per_second": 0.901,
	"step": 500
	},
	{
	"epoch": 0.7656220679301933,
	"grad_norm": 0.5762482285499573,
	"kl": 3.2514896392822266,
	"learning_rate": 1.4571910395143964e-05,
	"logits/chosen": -123910669.4736842,
	"logits/rejected": -82550308.57142857,
	"logps/chosen": -20.17646066766036,
	"logps/rejected": -92.69414411272321,
	"loss": 0.21347689628601074,
	"rewards/chosen": 1.0528185995001542,
	"rewards/margins": 6.815291206340742,
	"rewards/rejected": -5.762472606840587,
	"step": 510
	},
	{
	"epoch": 0.7806342653405892,
	"grad_norm": 0.6658563017845154,
	"kl": 3.7189126014709473,
	"learning_rate": 1.4350004050571093e-05,
	"logits/chosen": -96275552.0,
	"logits/rejected": -91569664.0,
	"logps/chosen": -24.03918914794922,
	"logps/rejected": -82.615673828125,
	"loss": 0.2262340307235718,
	"rewards/chosen": 1.1256752014160156,
	"rewards/margins": 6.1439353942871096,
	"rewards/rejected": -5.018260192871094,
	"step": 520
	},
	{
	"epoch": 0.7956464627509852,
	"grad_norm": 0.7213343977928162,
	"kl": 4.146698951721191,
	"learning_rate": 1.412542338258905e-05,
	"logits/chosen": -71005795.90243903,
	"logits/rejected": -80605669.74358974,
	"logps/chosen": -21.523729277820124,
	"logps/rejected": -77.89060621995192,
	"loss": 0.22721762657165528,
	"rewards/chosen": 1.1042013400938453,
	"rewards/margins": 6.000391885591641,
	"rewards/rejected": -4.896190545497796,
	"step": 530
	},
	{
	"epoch": 0.8106586601613811,
	"grad_norm": 0.6627029180526733,
	"kl": 4.362208366394043,
	"learning_rate": 1.3898306460342295e-05,
	"logits/chosen": -103514886.91891892,
	"logits/rejected": -70722119.44186047,
	"logps/chosen": -30.26540725295608,
	"logps/rejected": -79.42020008175872,
	"loss": 0.21057591438293458,
	"rewards/chosen": 1.2330357319599874,
	"rewards/margins": 6.344258110003228,
	"rewards/rejected": -5.111222378043241,
	"step": 540
	},
	{
	"epoch": 0.825670857571777,
	"grad_norm": 0.6026250720024109,
	"kl": 4.500165939331055,
	"learning_rate": 1.366879291222995e-05,
	"logits/chosen": -97725642.41860466,
	"logits/rejected": -77692900.32432432,
	"logps/chosen": -19.79853856286337,
	"logps/rejected": -86.36903505067568,
	"loss": 0.23624191284179688,
	"rewards/chosen": 1.1637908580691316,
	"rewards/margins": 6.564948546519301,
	"rewards/rejected": -5.401157688450169,
	"step": 550
	},
	{
	"epoch": 0.825670857571777,
	"eval_kl": 0.13565272092819214,
	"eval_logits/chosen": -52988113.08900524,
	"eval_logits/rejected": -37333402.136125654,
	"eval_logps/chosen": -25.6715835651178,
	"eval_logps/rejected": -88.75424370091623,
	"eval_loss": 0.16850697994232178,
	"eval_rewards/chosen": 1.1059383372361746,
	"eval_rewards/margins": 6.7323411871625485,
	"eval_rewards/rejected": -5.626402849926374,
	"eval_runtime": 436.4536,
	"eval_samples_per_second": 1.75,
	"eval_steps_per_second": 0.875,
	"step": 550
	},
	{
	"epoch": 0.840683054982173,
	"grad_norm": 0.5151755213737488,
	"kl": 3.809490203857422,
	"learning_rate": 1.3437023840064178e-05,
	"logits/chosen": -93870756.97777778,
	"logits/rejected": -93916525.71428572,
	"logps/chosen": -24.47983127170139,
	"logps/rejected": -81.66769670758929,
	"loss": 0.2295558214187622,
	"rewards/chosen": 1.177097405327691,
	"rewards/margins": 6.296527511354477,
	"rewards/rejected": -5.119430106026786,
	"step": 560
	},
	{
	"epoch": 0.8556952523925689,
	"grad_norm": 0.5849828124046326,
	"kl": 3.8631434440612793,
	"learning_rate": 1.3203141732322693e-05,
	"logits/chosen": -99688978.96296297,
	"logits/rejected": -75819280.20253165,
	"logps/chosen": -28.64499240451389,
	"logps/rejected": -93.65779519382912,
	"loss": 0.2268832206726074,
	"rewards/chosen": 1.1103750158239294,
	"rewards/margins": 7.005714402196555,
	"rewards/rejected": -5.895339386372626,
	"step": 570
	},
	{
	"epoch": 0.8707074498029649,
	"grad_norm": 0.6972293257713318,
	"kl": 4.144640922546387,
	"learning_rate": 1.2967290376548783e-05,
	"logits/chosen": -105536728.84705882,
	"logits/rejected": -104656288.42666666,
	"logps/chosen": -25.18662109375,
	"logps/rejected": -73.02625,
	"loss": 0.23414719104766846,
	"rewards/chosen": 1.136143762925092,
	"rewards/margins": 5.7962092739928,
	"rewards/rejected": -4.6600655110677085,
	"step": 580
	},
	{
	"epoch": 0.8857196472133608,
	"grad_norm": 0.6593893766403198,
	"kl": 4.090695858001709,
	"learning_rate": 1.2729614770952672e-05,
	"logits/chosen": -82203424.0,
	"logits/rejected": -89865196.8,
	"logps/chosen": -24.66193389892578,
	"logps/rejected": -81.38163452148437,
	"loss": 0.215281081199646,
	"rewards/chosen": 1.1623407363891602,
	"rewards/margins": 6.0649263381958,
	"rewards/rejected": -4.90258560180664,
	"step": 590
	},
	{
	"epoch": 0.9007318446237568,
	"grad_norm": 0.30154913663864136,
	"kl": 4.011666297912598,
	"learning_rate": 1.2490261035268614e-05,
	"logits/chosen": -103245324.64197531,
	"logits/rejected": -64040046.17721519,
	"logps/chosen": -25.815553264853396,
	"logps/rejected": -85.59282288370254,
	"loss": 0.22003612518310547,
	"rewards/chosen": 1.1088857768494407,
	"rewards/margins": 6.442291476760437,
	"rewards/rejected": -5.333405699910997,
	"step": 600
	},
	{
	"epoch": 0.9007318446237568,
	"eval_kl": 0.1486717015504837,
	"eval_logits/chosen": -52628898.17801047,
	"eval_logits/rejected": -37037149.82198953,
	"eval_logps/chosen": -25.418955538285342,
	"eval_logps/rejected": -87.40471613219896,
	"eval_loss": 0.1679808497428894,
	"eval_rewards/chosen": 1.131200820363629,
	"eval_rewards/margins": 6.622650146484375,
	"eval_rewards/rejected": -5.491449326120746,
	"eval_runtime": 434.8716,
	"eval_samples_per_second": 1.757,
	"eval_steps_per_second": 0.878,
	"step": 600
	},
	{
	"epoch": 0.9157440420341527,
	"grad_norm": 0.8136588931083679,
	"kl": 4.867345333099365,
	"learning_rate": 1.2249376320922435e-05,
	"logits/chosen": -96478865.38271604,
	"logits/rejected": -88826089.31645569,
	"logps/chosen": -24.912190755208332,
	"logps/rejected": -73.71896014636076,
	"loss": 0.2281566619873047,
	"rewards/chosen": 1.2302150961793499,
	"rewards/margins": 5.906620990784769,
	"rewards/rejected": -4.676405894605419,
	"step": 610
	},
	{
	"epoch": 0.9307562394445487,
	"grad_norm": 0.34996625781059265,
	"kl": 3.6859207153320312,
	"learning_rate": 1.2007108720564852e-05,
	"logits/chosen": -104111360.0,
	"logits/rejected": -78802962.73170732,
	"logps/chosen": -17.025127704326923,
	"logps/rejected": -87.1907274199695,
	"loss": 0.21024839878082274,
	"rewards/chosen": 1.2063480279384515,
	"rewards/margins": 6.545918236232088,
	"rewards/rejected": -5.339570208293636,
	"step": 620
	},
	{
	"epoch": 0.9457684368549446,
	"grad_norm": 0.5199890732765198,
	"kl": 4.800234794616699,
	"learning_rate": 1.1763607177026131e-05,
	"logits/chosen": -101240820.74725275,
	"logits/rejected": -65516358.492753625,
	"logps/chosen": -25.177771935096153,
	"logps/rejected": -84.0569095335145,
	"loss": 0.2449950933456421,
	"rewards/chosen": 1.216908716893458,
	"rewards/margins": 6.35318651000374,
	"rewards/rejected": -5.136277793110281,
	"step": 630
	},
	{
	"epoch": 0.9607806342653405,
	"grad_norm": 0.5771164298057556,
	"kl": 4.2080159187316895,
	"learning_rate": 1.1519021391748024e-05,
	"logits/chosen": -97543491.67816092,
	"logits/rejected": -95486464.0,
	"logps/chosen": -25.306859509698278,
	"logps/rejected": -79.20900444135275,
	"loss": 0.2462038516998291,
	"rewards/chosen": 1.1318982880691002,
	"rewards/margins": 6.078684828897439,
	"rewards/rejected": -4.946786540828339,
	"step": 640
	},
	{
	"epoch": 0.9757928316757365,
	"grad_norm": 0.6379005312919617,
	"kl": 4.442286014556885,
	"learning_rate": 1.12735017327494e-05,
	"logits/chosen": -89721952.46376811,
	"logits/rejected": -92305312.35164835,
	"logps/chosen": -26.444505774456523,
	"logps/rejected": -81.37484975961539,
	"loss": 0.20378758907318115,
	"rewards/chosen": 1.1361859086631,
	"rewards/margins": 6.409890353632193,
	"rewards/rejected": -5.273704444969093,
	"step": 650
	},
	{
	"epoch": 0.9757928316757365,
	"eval_kl": 0.14565975964069366,
	"eval_logits/chosen": -52615848.87958115,
	"eval_logits/rejected": -36491556.18848167,
	"eval_logps/chosen": -25.423930382853403,
	"eval_logps/rejected": -88.12435577552355,
	"eval_loss": 0.16747047007083893,
	"eval_rewards/chosen": 1.1307032719956642,
	"eval_rewards/margins": 6.694117201560454,
	"eval_rewards/rejected": -5.56341392956479,
	"eval_runtime": 433.8353,
	"eval_samples_per_second": 1.761,
	"eval_steps_per_second": 0.881,
	"step": 650
	},
	{
	"epoch": 0.9908050290861324,
	"grad_norm": 0.466814786195755,
	"kl": 5.0540056228637695,
	"learning_rate": 1.1027199142181999e-05,
	"logits/chosen": -101186585.6,
	"logits/rejected": -91657702.4,
	"logps/chosen": -21.362211608886717,
	"logps/rejected": -79.55250854492188,
	"loss": 0.23439102172851561,
	"rewards/chosen": 1.1953808784484863,
	"rewards/margins": 6.219569492340088,
	"rewards/rejected": -5.024188613891601,
	"step": 660
	},
	{
	"epoch": 1.0045036592231187,
	"grad_norm": 0.6553441286087036,
	"kl": 3.510671615600586,
	"learning_rate": 1.0780265043533252e-05,
	"logits/chosen": -106023401.07462686,
	"logits/rejected": -84727529.31645569,
	"logps/chosen": -23.81528575384795,
	"logps/rejected": -83.47320633900317,
	"loss": 0.21112267971038817,
	"rewards/chosen": 1.0875146211083255,
	"rewards/margins": 6.22628809260259,
	"rewards/rejected": -5.1387734714942646,
	"step": 670
	},
	{
	"epoch": 1.0195158566335147,
	"grad_norm": 0.7766548991203308,
	"kl": 4.419243812561035,
	"learning_rate": 1.0532851248533187e-05,
	"logits/chosen": -140482802.52631578,
	"logits/rejected": -84243260.95238096,
	"logps/chosen": -26.12240921823602,
	"logps/rejected": -79.62333170572917,
	"loss": 0.2175424337387085,
	"rewards/chosen": 1.0876518048738177,
	"rewards/margins": 6.136545324684086,
	"rewards/rejected": -5.048893519810268,
	"step": 680
	},
	{
	"epoch": 1.0345280540439106,
	"grad_norm": 0.6276580095291138,
	"kl": 3.7814064025878906,
	"learning_rate": 1.0285109863822632e-05,
	"logits/chosen": -85505976.70886075,
	"logits/rejected": -103004653.03703703,
	"logps/chosen": -25.645328570015824,
	"logps/rejected": -85.05873239776234,
	"loss": 0.22568042278289796,
	"rewards/chosen": 1.1665521935571599,
	"rewards/margins": 6.489668543142423,
	"rewards/rejected": -5.323116349585263,
	"step": 690
	},
	{
	"epoch": 1.0495402514543066,
	"grad_norm": 0.4705566167831421,
	"kl": 4.468962669372559,
	"learning_rate": 1.0037193197440117e-05,
	"logits/chosen": -93083011.28205128,
	"logits/rejected": -83736931.90243903,
	"logps/chosen": -26.694182567107372,
	"logps/rejected": -77.2444502667683,
	"loss": 0.23378393650054932,
	"rewards/chosen": 1.1905487256172376,
	"rewards/margins": 6.1078944704247835,
	"rewards/rejected": -4.9173457448075455,
	"step": 700
	},
	{
	"epoch": 1.0495402514543066,
	"eval_kl": 0.13426250219345093,
	"eval_logits/chosen": -53473505.172774866,
	"eval_logits/rejected": -37231283.60209424,
	"eval_logps/chosen": -25.48450537876309,
	"eval_logps/rejected": -88.80958565117801,
	"eval_loss": 0.1667330414056778,
	"eval_rewards/chosen": 1.1246460120715396,
	"eval_rewards/margins": 6.756583568313359,
	"eval_rewards/rejected": -5.6319375562418195,
	"eval_runtime": 433.1925,
	"eval_samples_per_second": 1.764,
	"eval_steps_per_second": 0.882,
	"step": 700
	},
	{
	"epoch": 1.0645524488647027,
	"grad_norm": 0.4909619390964508,
	"kl": 3.246283769607544,
	"learning_rate": 9.789253665184971e-06,
	"logits/chosen": -87369889.3150685,
	"logits/rejected": -85285476.04597701,
	"logps/chosen": -21.669224569242296,
	"logps/rejected": -90.8365548670977,
	"loss": 0.19179887771606446,
	"rewards/chosen": 1.123171035557577,
	"rewards/margins": 6.876017238188684,
	"rewards/rejected": -5.7528462026311065,
	"step": 710
	},
	{
	"epoch": 1.0795646462750985,
	"grad_norm": 0.5964953899383545,
	"kl": 4.596374034881592,
	"learning_rate": 9.541443696914142e-06,
	"logits/chosen": -87436251.85882352,
	"logits/rejected": -91590417.06666666,
	"logps/chosen": -19.859073414522058,
	"logps/rejected": -85.19293619791667,
	"loss": 0.24003658294677735,
	"rewards/chosen": 1.0800759708180148,
	"rewards/margins": 6.487529991000306,
	"rewards/rejected": -5.407454020182292,
	"step": 720
	},
	{
	"epoch": 1.0945768436854943,
	"grad_norm": 0.7163007855415344,
	"kl": 4.345240592956543,
	"learning_rate": 9.293915642830407e-06,
	"logits/chosen": -100356619.37777779,
	"logits/rejected": -67504727.77142857,
	"logps/chosen": -23.59975043402778,
	"logps/rejected": -85.96699916294642,
	"loss": 0.24338247776031494,
	"rewards/chosen": 1.1390112135145398,
	"rewards/margins": 6.603775617811415,
	"rewards/rejected": -5.464764404296875,
	"step": 730
	},
	{
	"epoch": 1.1095890410958904,
	"grad_norm": 0.4801461398601532,
	"kl": 4.036700248718262,
	"learning_rate": 9.046821679819528e-06,
	"logits/chosen": -99535643.75903614,
	"logits/rejected": -75779311.37662338,
	"logps/chosen": -23.405419039439007,
	"logps/rejected": -80.26659522118507,
	"loss": 0.22335414886474608,
	"rewards/chosen": 1.214461085308029,
	"rewards/margins": 6.408107801141024,
	"rewards/rejected": -5.193646715832995,
	"step": 740
	},
	{
	"epoch": 1.1246012385062865,
	"grad_norm": 0.8056963682174683,
	"kl": 3.960627794265747,
	"learning_rate": 8.800313717893959e-06,
	"logits/chosen": -105539597.4736842,
	"logits/rejected": -83317174.85714285,
	"logps/chosen": -28.418807180304277,
	"logps/rejected": -83.92525809151786,
	"loss": 0.18643591403961182,
	"rewards/chosen": 1.4098168423301296,
	"rewards/margins": 6.669779015065434,
	"rewards/rejected": -5.259962172735305,
	"step": 750
	},
	{
	"epoch": 1.1246012385062865,
	"eval_kl": 0.13305240869522095,
	"eval_logits/chosen": -54042077.15183246,
	"eval_logits/rejected": -37955905.67539267,
	"eval_logps/chosen": -25.507919870746072,
	"eval_logps/rejected": -89.86107248036649,
	"eval_loss": 0.16618458926677704,
	"eval_rewards/chosen": 1.1223043871175555,
	"eval_rewards/margins": 6.859389220232739,
	"eval_rewards/rejected": -5.737084833115183,
	"eval_runtime": 433.0131,
	"eval_samples_per_second": 1.764,
	"eval_steps_per_second": 0.882,
	"step": 750
	},
	{
	"epoch": 1.1396134359166823,
	"grad_norm": 0.5588005781173706,
	"kl": 3.8682758808135986,
	"learning_rate": 8.554543306800668e-06,
	"logits/chosen": -115601833.55844156,
	"logits/rejected": -101657297.73493975,
	"logps/chosen": -26.970395571225648,
	"logps/rejected": -90.736375188253,
	"loss": 0.20916638374328614,
	"rewards/chosen": 1.233702325201654,
	"rewards/margins": 6.799562200248717,
	"rewards/rejected": -5.565859875047063,
	"step": 760
	},
	{
	"epoch": 1.1546256333270781,
	"grad_norm": 0.6878859996795654,
	"kl": 3.435823917388916,
	"learning_rate": 8.309661542850376e-06,
	"logits/chosen": -85981894.4,
	"logits/rejected": -88435206.4,
	"logps/chosen": -18.470010375976564,
	"logps/rejected": -87.00191650390624,
	"loss": 0.21931285858154298,
	"rewards/chosen": 1.0244508743286134,
	"rewards/margins": 6.513032722473145,
	"rewards/rejected": -5.488581848144531,
	"step": 770
	},
	{
	"epoch": 1.1696378307374742,
	"grad_norm": 0.6787204742431641,
	"kl": 4.617588520050049,
	"learning_rate": 8.065818976025607e-06,
	"logits/chosen": -100501036.8,
	"logits/rejected": -93501510.4,
	"logps/chosen": -23.78038635253906,
	"logps/rejected": -80.3609375,
	"loss": 0.22347478866577147,
	"rewards/chosen": 1.2114054679870605,
	"rewards/margins": 6.213579082489014,
	"rewards/rejected": -5.002173614501953,
	"step": 780
	},
	{
	"epoch": 1.1846500281478702,
	"grad_norm": 0.8261892795562744,
	"kl": 4.476070404052734,
	"learning_rate": 7.823165517424608e-06,
	"logits/chosen": -109933940.36363636,
	"logits/rejected": -87284124.44444445,
	"logps/chosen": -26.592254638671875,
	"logps/rejected": -86.63107638888889,
	"loss": 0.24246513843536377,
	"rewards/chosen": 1.1668794805353337,
	"rewards/margins": 6.552800534951567,
	"rewards/rejected": -5.385921054416233,
	"step": 790
	},
	{
	"epoch": 1.199662225558266,
	"grad_norm": 0.6302554607391357,
	"kl": 4.160749435424805,
	"learning_rate": 7.581850347098052e-06,
	"logits/chosen": -106630573.41935484,
	"logits/rejected": -80972089.31343284,
	"logps/chosen": -22.595963016633064,
	"logps/rejected": -93.17525652985074,
	"loss": 0.26153428554534913,
	"rewards/chosen": 0.970795416062878,
	"rewards/margins": 6.776746890380742,
	"rewards/rejected": -5.8059514743178635,
	"step": 800
	},
	{
	"epoch": 1.199662225558266,
	"eval_kl": 0.1418178677558899,
	"eval_logits/chosen": -54769111.790575914,
	"eval_logits/rejected": -38202357.27748691,
	"eval_logps/chosen": -25.350251043030106,
	"eval_logps/rejected": -89.12186068390052,
	"eval_loss": 0.16583210229873657,
	"eval_rewards/chosen": 1.1380715095559963,
	"eval_rewards/margins": 6.801235418669217,
	"eval_rewards/rejected": -5.66316390911322,
	"eval_runtime": 439.3783,
	"eval_samples_per_second": 1.739,
	"eval_steps_per_second": 0.869,
	"step": 800
	},
	{
	"epoch": 1.2146744229686621,
	"grad_norm": 0.866295337677002,
	"kl": 3.919668197631836,
	"learning_rate": 7.342021822335144e-06,
	"logits/chosen": -100262202.02666667,
	"logits/rejected": -68069791.6235294,
	"logps/chosen": -35.465442708333335,
	"logps/rejected": -78.75108570772059,
	"loss": 0.2064718246459961,
	"rewards/chosen": 1.1432391357421876,
	"rewards/margins": 6.094182667451746,
	"rewards/rejected": -4.950943531709559,
	"step": 810
	},
	{
	"epoch": 1.229686620379058,
	"grad_norm": 0.5769370198249817,
	"kl": 4.099992275238037,
	"learning_rate": 7.1038273864555865e-06,
	"logits/chosen": -97576707.0117647,
	"logits/rejected": -107417941.33333333,
	"logps/chosen": -25.98314280790441,
	"logps/rejected": -87.1881640625,
	"loss": 0.23273687362670897,
	"rewards/chosen": 1.1329481237074908,
	"rewards/margins": 6.32427869011374,
	"rewards/rejected": -5.19133056640625,
	"step": 820
	},
	{
	"epoch": 1.244698817789454,
	"grad_norm": 0.7531806230545044,
	"kl": 3.6843295097351074,
	"learning_rate": 6.867413478163445e-06,
	"logits/chosen": -78316946.28571428,
	"logits/rejected": -93422969.26315789,
	"logps/chosen": -23.63427007765997,
	"logps/rejected": -83.31368857935855,
	"loss": 0.23035976886749268,
	"rewards/chosen": 1.1488791874476842,
	"rewards/margins": 6.381895352126961,
	"rewards/rejected": -5.2330161646792765,
	"step": 830
	},
	{
	"epoch": 1.2597110151998498,
	"grad_norm": 0.6657658219337463,
	"kl": 3.697706699371338,
	"learning_rate": 6.632925441518594e-06,
	"logits/chosen": -121944169.02564102,
	"logits/rejected": -104590610.73170732,
	"logps/chosen": -25.239830603966347,
	"logps/rejected": -85.82620760289635,
	"loss": 0.20410506725311278,
	"rewards/chosen": 1.2310838943872697,
	"rewards/margins": 6.589877571740548,
	"rewards/rejected": -5.358793677353278,
	"step": 840
	},
	{
	"epoch": 1.274723212610246,
	"grad_norm": 0.718334972858429,
	"kl": 3.1744580268859863,
	"learning_rate": 6.400507436581178e-06,
	"logits/chosen": -118265521.77777778,
	"logits/rejected": -87497675.63636364,
	"logps/chosen": -22.66006130642361,
	"logps/rejected": -80.14004239169034,
	"loss": 0.2051610231399536,
	"rewards/chosen": 1.1418331993950739,
	"rewards/margins": 6.250424799292979,
	"rewards/rejected": -5.108591599897905,
	"step": 850
	},
	{
	"epoch": 1.274723212610246,
	"eval_kl": 0.13580438494682312,
	"eval_logits/chosen": -54494524.314136125,
	"eval_logits/rejected": -38059876.52356021,
	"eval_logps/chosen": -25.360545852421467,
	"eval_logps/rejected": -90.01760880235602,
	"eval_loss": 0.1653730720281601,
	"eval_rewards/chosen": 1.1370419806834915,
	"eval_rewards/margins": 6.889781212931529,
	"eval_rewards/rejected": -5.752739232248037,
	"eval_runtime": 436.4699,
	"eval_samples_per_second": 1.75,
	"eval_steps_per_second": 0.875,
	"step": 850
	},
	{
	"epoch": 1.2897354100206417,
	"grad_norm": 0.8406268954277039,
	"kl": 3.3996193408966064,
	"learning_rate": 6.17030235078397e-06,
	"logits/chosen": -103157228.05194806,
	"logits/rejected": -76576545.92771085,
	"logps/chosen": -23.79248046875,
	"logps/rejected": -83.25042945218374,
	"loss": 0.1881808042526245,
	"rewards/chosen": 1.337797784186029,
	"rewards/margins": 6.671168866170216,
	"rewards/rejected": -5.333371081984187,
	"step": 860
	},
	{
	"epoch": 1.3047476074310378,
	"grad_norm": 0.5345896482467651,
	"kl": 4.3026533126831055,
	"learning_rate": 5.942451711087103e-06,
	"logits/chosen": -91860792.19512194,
	"logits/rejected": -88229369.43589744,
	"logps/chosen": -25.162978753810975,
	"logps/rejected": -85.17337114383012,
	"loss": 0.2208491086959839,
	"rewards/chosen": 1.286998004448123,
	"rewards/margins": 6.5604120293880275,
	"rewards/rejected": -5.273414024939904,
	"step": 870
	},
	{
	"epoch": 1.3197598048414336,
	"grad_norm": 0.774782121181488,
	"kl": 4.132607460021973,
	"learning_rate": 5.717095596969226e-06,
	"logits/chosen": -90163577.26315789,
	"logits/rejected": -97945203.8095238,
	"logps/chosen": -25.397158974095394,
	"logps/rejected": -81.13313220796131,
	"loss": 0.23573668003082277,
	"rewards/chosen": 1.0274274725663035,
	"rewards/margins": 6.3404724293185355,
	"rewards/rejected": -5.313044956752232,
	"step": 880
	},
	{
	"epoch": 1.3347720022518297,
	"grad_norm": 0.7283003926277161,
	"kl": 4.143404006958008,
	"learning_rate": 5.494372554308554e-06,
	"logits/chosen": -98379586.37037037,
	"logits/rejected": -108482605.3670886,
	"logps/chosen": -26.02851019965278,
	"logps/rejected": -84.22099362143987,
	"loss": 0.2245171546936035,
	"rewards/chosen": 1.1866206416377314,
	"rewards/margins": 6.487602380537059,
	"rewards/rejected": -5.300981738899328,
	"step": 890
	},
	{
	"epoch": 1.3497841996622255,
	"grad_norm": 0.8141359686851501,
	"kl": 2.8307042121887207,
	"learning_rate": 5.274419510206713e-06,
	"logits/chosen": -99083153.2972973,
	"logits/rejected": -92244980.09302326,
	"logps/chosen": -26.87116303315034,
	"logps/rejected": -93.76043559229652,
	"loss": 0.1964324712753296,
	"rewards/chosen": 1.0808814796241555,
	"rewards/margins": 7.159457855895957,
	"rewards/rejected": -6.078576376271802,
	"step": 900
	},
	{
	"epoch": 1.3497841996622255,
	"eval_kl": 0.13515853881835938,
	"eval_logits/chosen": -55153846.28272251,
	"eval_logits/rejected": -38913712.92146597,
	"eval_logps/chosen": -25.354182857493456,
	"eval_logps/rejected": -90.31069003599477,
	"eval_loss": 0.16496765613555908,
	"eval_rewards/chosen": 1.1376782162651342,
	"eval_rewards/margins": 6.919725188410094,
	"eval_rewards/rejected": -5.78204697214496,
	"eval_runtime": 421.2799,
	"eval_samples_per_second": 1.814,
	"eval_steps_per_second": 0.907,
	"step": 900
	},
	{
	"epoch": 1.3647963970726216,
	"grad_norm": 0.7665517926216125,
	"kl": 4.347356796264648,
	"learning_rate": 5.057371688807849e-06,
	"logits/chosen": -97613490.08695652,
	"logits/rejected": -81990761.41176471,
	"logps/chosen": -20.336362092391305,
	"logps/rejected": -81.53354061351104,
	"loss": 0.2507650375366211,
	"rewards/chosen": 1.0822478584621265,
	"rewards/margins": 6.298613897065067,
	"rewards/rejected": -5.216366038602941,
	"step": 910
	},
	{
	"epoch": 1.3798085944830174,
	"grad_norm": 0.7740744352340698,
	"kl": 3.4625542163848877,
	"learning_rate": 4.843362528164653e-06,
	"logits/chosen": -89836807.1111111,
	"logits/rejected": -76998423.27272727,
	"logps/chosen": -26.65255567762587,
	"logps/rejected": -82.26154119318181,
	"loss": 0.19089690446853638,
	"rewards/chosen": 1.2612567477756076,
	"rewards/margins": 6.494063907199436,
	"rewards/rejected": -5.232807159423828,
	"step": 920
	},
	{
	"epoch": 1.3948207918934135,
	"grad_norm": 0.9630061984062195,
	"kl": 5.1812615394592285,
	"learning_rate": 4.632523598202435e-06,
	"logits/chosen": -93226178.56,
	"logits/rejected": -92916608.0,
	"logps/chosen": -22.01804443359375,
	"logps/rejected": -70.4688232421875,
	"loss": 0.2744077205657959,
	"rewards/chosen": 1.1364972686767578,
	"rewards/margins": 5.4957613881429035,
	"rewards/rejected": -4.359264119466146,
	"step": 930
	},
	{
	"epoch": 1.4098329893038093,
	"grad_norm": 0.6798044443130493,
	"kl": 3.6221089363098145,
	"learning_rate": 4.4249845198317595e-06,
	"logits/chosen": -105949212.05479452,
	"logits/rejected": -94108677.88505746,
	"logps/chosen": -25.15871147260274,
	"logps/rejected": -91.54803116020115,
	"loss": 0.1968392014503479,
	"rewards/chosen": 1.132062206529591,
	"rewards/margins": 7.05869023162837,
	"rewards/rejected": -5.926628025098779,
	"step": 940
	},
	{
	"epoch": 1.4248451867142053,
	"grad_norm": 0.8968099355697632,
	"kl": 4.054515838623047,
	"learning_rate": 4.2208728852592475e-06,
	"logits/chosen": -93645798.4,
	"logits/rejected": -91873120.0,
	"logps/chosen": -30.97816162109375,
	"logps/rejected": -87.55768432617188,
	"loss": 0.20583555698394776,
	"rewards/chosen": 1.347819995880127,
	"rewards/margins": 7.024199390411377,
	"rewards/rejected": -5.67637939453125,
	"step": 950
	},
	{
	"epoch": 1.4248451867142053,
	"eval_kl": 0.13550503551959991,
	"eval_logits/chosen": -55356973.570680626,
	"eval_logits/rejected": -39265502.4921466,
	"eval_logps/chosen": -25.34827491001309,
	"eval_logps/rejected": -90.32589577879581,
	"eval_loss": 0.16480737924575806,
	"eval_rewards/chosen": 1.1382690749243292,
	"eval_rewards/margins": 6.921837132638662,
	"eval_rewards/rejected": -5.783568057714333,
	"eval_runtime": 421.7775,
	"eval_samples_per_second": 1.811,
	"eval_steps_per_second": 0.906,
	"step": 950
	},
	{
	"epoch": 1.4398573841246012,
	"grad_norm": 0.9420547485351562,
	"kl": 3.70204496383667,
	"learning_rate": 4.020314179545609e-06,
	"logits/chosen": -85311225.75609756,
	"logits/rejected": -102413193.84615384,
	"logps/chosen": -23.1737060546875,
	"logps/rejected": -92.63661232972757,
	"loss": 0.22018556594848632,
	"rewards/chosen": 1.1600302719488376,
	"rewards/margins": 6.917001582295392,
	"rewards/rejected": -5.756971310346555,
	"step": 960
	},
	{
	"epoch": 1.4548695815349972,
	"grad_norm": 0.6381933689117432,
	"kl": 3.5098605155944824,
	"learning_rate": 3.823431703459164e-06,
	"logits/chosen": -91596382.31578948,
	"logits/rejected": -75453647.23809524,
	"logps/chosen": -22.27265046772204,
	"logps/rejected": -97.9072265625,
	"loss": 0.19860419034957885,
	"rewards/chosen": 1.1405779186048006,
	"rewards/margins": 7.267270917581735,
	"rewards/rejected": -6.126692998976934,
	"step": 970
	},
	{
	"epoch": 1.469881778945393,
	"grad_norm": 0.801608145236969,
	"kl": 4.244645118713379,
	"learning_rate": 3.6303464976721835e-06,
	"logits/chosen": -83721495.81395349,
	"logits/rejected": -104408437.62162162,
	"logps/chosen": -23.19318336664244,
	"logps/rejected": -76.96543760557432,
	"loss": 0.23014593124389648,
	"rewards/chosen": 1.1628135858580124,
	"rewards/margins": 6.100889708245048,
	"rewards/rejected": -4.938076122387035,
	"step": 980
	},
	{
	"epoch": 1.4848939763557891,
	"grad_norm": 0.6242425441741943,
	"kl": 4.364640712738037,
	"learning_rate": 3.4411772683467235e-06,
	"logits/chosen": -100387865.92405063,
	"logits/rejected": -97113922.37037037,
	"logps/chosen": -22.812269766119464,
	"logps/rejected": -83.23450159143519,
	"loss": 0.20972530841827391,
	"rewards/chosen": 1.2276213440714003,
	"rewards/margins": 6.537462170412065,
	"rewards/rejected": -5.309840826340664,
	"step": 990
	},
	{
	"epoch": 1.499906173766185,
	"grad_norm": 0.6481726765632629,
	"kl": 3.0429587364196777,
	"learning_rate": 3.2560403141557085e-06,
	"logits/chosen": -108133474.19178082,
	"logits/rejected": -89296595.86206897,
	"logps/chosen": -25.251272541202912,
	"logps/rejected": -89.57416823814656,
	"loss": 0.19645323753356933,
	"rewards/chosen": 1.1944668913540775,
	"rewards/margins": 7.028910433132992,
	"rewards/rejected": -5.834443541778915,
	"step": 1000
	},
	{
	"epoch": 1.499906173766185,
	"eval_kl": 0.12928542494773865,
	"eval_logits/chosen": -55498553.633507855,
	"eval_logits/rejected": -39448407.12041885,
	"eval_logps/chosen": -25.369434616328533,
	"eval_logps/rejected": -90.84591786649214,
	"eval_loss": 0.16446402668952942,
	"eval_rewards/chosen": 1.13615289658152,
	"eval_rewards/margins": 6.971722268309269,
	"eval_rewards/rejected": -5.835569371727749,
	"eval_runtime": 422.5452,
	"eval_samples_per_second": 1.808,
	"eval_steps_per_second": 0.904,
	"step": 1000
	},
	{
	"epoch": 1.514918371176581,
	"grad_norm": 0.8008208274841309,
	"kl": 4.670881271362305,
	"learning_rate": 3.0750494547840737e-06,
	"logits/chosen": -125810654.24175824,
	"logits/rejected": -89304412.75362319,
	"logps/chosen": -21.928126073145606,
	"logps/rejected": -89.32652570199275,
	"loss": 0.24952163696289062,
	"rewards/chosen": 1.1456220018994676,
	"rewards/margins": 6.920434119766315,
	"rewards/rejected": -5.774812117866848,
	"step": 1010
	},
	{
	"epoch": 1.5299305685869768,
	"grad_norm": 0.7026108503341675,
	"kl": 3.7687172889709473,
	"learning_rate": 2.8983159609539634e-06,
	"logits/chosen": -110794227.2,
	"logits/rejected": -108752460.8,
	"logps/chosen": -27.65631103515625,
	"logps/rejected": -90.34964599609376,
	"loss": 0.21506090164184571,
	"rewards/chosen": 1.2219769477844238,
	"rewards/margins": 6.7797516822814945,
	"rewards/rejected": -5.5577747344970705,
	"step": 1020
	},
	{
	"epoch": 1.544942765997373,
	"grad_norm": 0.7361843585968018,
	"kl": 5.285219669342041,
	"learning_rate": 2.7259484860170206e-06,
	"logits/chosen": -122753947.82608695,
	"logits/rejected": -96830245.64705883,
	"logps/chosen": -23.716366975203805,
	"logps/rejected": -79.43409639246323,
	"loss": 0.25809249877929685,
	"rewards/chosen": 1.178453196649966,
	"rewards/margins": 6.409470365480389,
	"rewards/rejected": -5.231017168830423,
	"step": 1030
	},
	{
	"epoch": 1.559954963407769,
	"grad_norm": 0.49050334095954895,
	"kl": 3.2523837089538574,
	"learning_rate": 2.558052999155768e-06,
	"logits/chosen": -92674758.68656716,
	"logits/rejected": -81995770.49462366,
	"logps/chosen": -23.637330923507463,
	"logps/rejected": -95.77403603830645,
	"loss": 0.16697038412094117,
	"rewards/chosen": 1.3456653196420243,
	"rewards/margins": 7.250253588291218,
	"rewards/rejected": -5.904588268649194,
	"step": 1040
	},
	{
	"epoch": 1.5749671608181648,
	"grad_norm": 0.872729480266571,
	"kl": 4.101504802703857,
	"learning_rate": 2.3947327202351887e-06,
	"logits/chosen": -114927466.14634146,
	"logits/rejected": -84980342.15384616,
	"logps/chosen": -27.76390410632622,
	"logps/rejected": -88.0575420673077,
	"loss": 0.2262244701385498,
	"rewards/chosen": 1.1740545877596227,
	"rewards/margins": 6.741694452167675,
	"rewards/rejected": -5.5676398644080525,
	"step": 1050
	},
	{
	"epoch": 1.5749671608181648,
	"eval_kl": 0.13142813742160797,
	"eval_logits/chosen": -55718252.56544503,
	"eval_logits/rejected": -39580859.64397906,
	"eval_logps/chosen": -25.34311088841623,
	"eval_logps/rejected": -90.90671016034031,
	"eval_loss": 0.16429825127124786,
	"eval_rewards/chosen": 1.1387850776392752,
	"eval_rewards/margins": 6.980434317863424,
	"eval_rewards/rejected": -5.841649240224149,
	"eval_runtime": 421.6304,
	"eval_samples_per_second": 1.812,
	"eval_steps_per_second": 0.906,
	"step": 1050
	},
	{
	"epoch": 1.5899793582285606,
	"grad_norm": 0.7624918222427368,
	"kl": 4.680811882019043,
	"learning_rate": 2.2360880563445763e-06,
	"logits/chosen": -109705061.20930232,
	"logits/rejected": -81854505.51351352,
	"logps/chosen": -22.512400072674417,
	"logps/rejected": -72.15485114020271,
	"loss": 0.2276033878326416,
	"rewards/chosen": 1.2336682164391806,
	"rewards/margins": 5.973305849066925,
	"rewards/rejected": -4.739637632627745,
	"step": 1060
	},
	{
	"epoch": 1.6049915556389567,
	"grad_norm": 0.4747796654701233,
	"kl": 3.267906904220581,
	"learning_rate": 2.0822165400686033e-06,
	"logits/chosen": -104269369.6,
	"logits/rejected": -106300960.0,
	"logps/chosen": -30.304464721679686,
	"logps/rejected": -98.04397583007812,
	"loss": 0.19671732187271118,
	"rewards/chosen": 1.3054424285888673,
	"rewards/margins": 7.5222820281982425,
	"rewards/rejected": -6.216839599609375,
	"step": 1070
	},
	{
	"epoch": 1.6200037530493527,
	"grad_norm": 0.9367624521255493,
	"kl": 3.2648837566375732,
	"learning_rate": 1.933212769525613e-06,
	"logits/chosen": -89395676.44444445,
	"logits/rejected": -104650240.0,
	"logps/chosen": -21.667002360026043,
	"logps/rejected": -84.4835371537642,
	"loss": 0.20028886795043946,
	"rewards/chosen": 1.037461280822754,
	"rewards/margins": 6.473727399652654,
	"rewards/rejected": -5.4362661188299,
	"step": 1080
	},
	{
	"epoch": 1.6350159504597486,
	"grad_norm": 0.6833193302154541,
	"kl": 3.370619297027588,
	"learning_rate": 1.7891683502099832e-06,
	"logits/chosen": -98081811.2,
	"logits/rejected": -81132697.6,
	"logps/chosen": -23.849533081054688,
	"logps/rejected": -80.27091674804687,
	"loss": 0.2054957628250122,
	"rewards/chosen": 1.2119413375854493,
	"rewards/margins": 6.449519538879395,
	"rewards/rejected": -5.237578201293945,
	"step": 1090
	},
	{
	"epoch": 1.6500281478701444,
	"grad_norm": 0.6855626702308655,
	"kl": 3.7833855152130127,
	"learning_rate": 1.6501718386743282e-06,
	"logits/chosen": -90577152.0,
	"logits/rejected": -74229938.60465117,
	"logps/chosen": -25.44064743454392,
	"logps/rejected": -92.05196788699128,
	"loss": 0.18979684114456177,
	"rewards/chosen": 1.2005912162162162,
	"rewards/margins": 7.079025153466398,
	"rewards/rejected": -5.878433937250182,
	"step": 1100
	},
	{
	"epoch": 1.6500281478701444,
	"eval_kl": 0.12844021618366241,
	"eval_logits/chosen": -56039509.78010471,
	"eval_logits/rejected": -39965537.84293193,
	"eval_logps/chosen": -25.35427488956152,
	"eval_logps/rejected": -91.16684391361257,
	"eval_loss": 0.16429360210895538,
	"eval_rewards/chosen": 1.1376691089250655,
	"eval_rewards/margins": 7.0053321079433895,
	"eval_rewards/rejected": -5.867662999018324,
	"eval_runtime": 421.252,
	"eval_samples_per_second": 1.814,
	"eval_steps_per_second": 0.907,
	"step": 1100
	},
	{
	"epoch": 1.6650403452805405,
	"grad_norm": 1.1428500413894653,
	"kl": 4.261361122131348,
	"learning_rate": 1.5163086880861045e-06,
	"logits/chosen": -90409088.0,
	"logits/rejected": -88890406.4,
	"logps/chosen": -27.13056640625,
	"logps/rejected": -81.66841430664063,
	"loss": 0.2300961971282959,
	"rewards/chosen": 1.1836955070495605,
	"rewards/margins": 6.292539882659912,
	"rewards/rejected": -5.108844375610351,
	"step": 1110
	},
	{
	"epoch": 1.6800525426909365,
	"grad_norm": 0.7117341160774231,
	"kl": 3.3029885292053223,
	"learning_rate": 1.3876611956921793e-06,
	"logits/chosen": -111789738.66666667,
	"logits/rejected": -96844811.63636364,
	"logps/chosen": -29.337073432074654,
	"logps/rejected": -84.19040749289773,
	"loss": 0.1927316427230835,
	"rewards/chosen": 1.232086393568251,
	"rewards/margins": 6.685429524893713,
	"rewards/rejected": -5.453343131325462,
	"step": 1120
	},
	{
	"epoch": 1.6950647401013323,
	"grad_norm": 0.5888846516609192,
	"kl": 3.230769395828247,
	"learning_rate": 1.264308452223616e-06,
	"logits/chosen": -111363690.98507462,
	"logits/rejected": -93282744.43010753,
	"logps/chosen": -23.554046175373134,
	"logps/rejected": -93.44833669354838,
	"loss": 0.17531968355178834,
	"rewards/chosen": 1.087284258942106,
	"rewards/margins": 7.054321993107093,
	"rewards/rejected": -5.967037734164987,
	"step": 1130
	},
	{
	"epoch": 1.7100769375117282,
	"grad_norm": 0.6945241093635559,
	"kl": 3.231876850128174,
	"learning_rate": 1.1463262932717323e-06,
	"logits/chosen": -100743661.26829268,
	"logits/rejected": -99711829.33333333,
	"logps/chosen": -25.812577410442074,
	"logps/rejected": -81.0398199619391,
	"loss": 0.21396026611328126,
	"rewards/chosen": 1.2160039762171304,
	"rewards/margins": 6.219116769185881,
	"rewards/rejected": -5.00311279296875,
	"step": 1140
	},
	{
	"epoch": 1.7250891349221242,
	"grad_norm": 0.7128337621688843,
	"kl": 2.7168688774108887,
	"learning_rate": 1.0337872526654357e-06,
	"logits/chosen": -115620558.22222222,
	"logits/rejected": -87219781.81818181,
	"logps/chosen": -23.565806070963543,
	"logps/rejected": -90.92021040482955,
	"loss": 0.1860736846923828,
	"rewards/chosen": 1.1724183824327257,
	"rewards/margins": 6.89070298936632,
	"rewards/rejected": -5.718284606933594,
	"step": 1150
	},
	{
	"epoch": 1.7250891349221242,
	"eval_kl": 0.12663534283638,
	"eval_logits/chosen": -56202390.11518325,
	"eval_logits/rejected": -40215629.738219894,
	"eval_logps/chosen": -25.382503169993456,
	"eval_logps/rejected": -91.49146146924083,
	"eval_loss": 0.16411779820919037,
	"eval_rewards/chosen": 1.134846392726399,
	"eval_rewards/margins": 7.034971147307552,
	"eval_rewards/rejected": -5.900124754581152,
	"eval_runtime": 419.4694,
	"eval_samples_per_second": 1.821,
	"eval_steps_per_second": 0.911,
	"step": 1150
	},
	{
	"epoch": 1.7401013323325203,
	"grad_norm": 0.7305581569671631,
	"kl": 3.2470200061798096,
	"learning_rate": 9.267605178784033e-07,
	"logits/chosen": -93602140.44444445,
	"logits/rejected": -86012282.18181819,
	"logps/chosen": -22.607952541775173,
	"logps/rejected": -88.2861328125,
	"loss": 0.1894938588142395,
	"rewards/chosen": 1.1574607425265842,
	"rewards/margins": 6.9352643369424225,
	"rewards/rejected": -5.777803594415838,
	"step": 1160
	},
	{
	"epoch": 1.7551135297429161,
	"grad_norm": 0.7399213314056396,
	"kl": 3.665466785430908,
	"learning_rate": 8.253118874935628e-07,
	"logits/chosen": -131597628.04938272,
	"logits/rejected": -97115894.278481,
	"logps/chosen": -22.83131992669753,
	"logps/rejected": -88.38763350474683,
	"loss": 0.22316858768463135,
	"rewards/chosen": 1.0157588440694927,
	"rewards/margins": 6.545139545834722,
	"rewards/rejected": -5.52938070176523,
	"step": 1170
	},
	{
	"epoch": 1.770125727153312,
	"grad_norm": 0.8386373519897461,
	"kl": 3.3606925010681152,
	"learning_rate": 7.2950373075101e-07,
	"logits/chosen": -123296072.20512821,
	"logits/rejected": -92810976.7804878,
	"logps/chosen": -24.13685803535657,
	"logps/rejected": -94.89799685594512,
	"loss": 0.2036738395690918,
	"rewards/chosen": 1.2165361062074318,
	"rewards/margins": 7.279486426567569,
	"rewards/rejected": -6.062950320360137,
	"step": 1180
	},
	{
	"epoch": 1.785137924563708,
	"grad_norm": 0.5710262656211853,
	"kl": 3.399351119995117,
	"learning_rate": 6.393949492042606e-07,
	"logits/chosen": -89650777.6,
	"logits/rejected": -79596396.8,
	"logps/chosen": -20.27527313232422,
	"logps/rejected": -84.6680419921875,
	"loss": 0.20696897506713868,
	"rewards/chosen": 1.1883172035217284,
	"rewards/margins": 6.458553409576416,
	"rewards/rejected": -5.270236206054688,
	"step": 1190
	},
	{
	"epoch": 1.800150121974104,
	"grad_norm": 0.7187512516975403,
	"kl": 4.214442729949951,
	"learning_rate": 5.550409405083656e-07,
	"logits/chosen": -86440611.84,
	"logits/rejected": -87346730.16470589,
	"logps/chosen": -23.561062825520832,
	"logps/rejected": -78.86371783088235,
	"loss": 0.21113624572753906,
	"rewards/chosen": 1.2850724283854167,
	"rewards/margins": 6.377582337622549,
	"rewards/rejected": -5.092509909237132,
	"step": 1200
	},
	{
	"epoch": 1.800150121974104,
	"eval_kl": 0.12559062242507935,
	"eval_logits/chosen": -56251209.71727749,
	"eval_logits/rejected": -40232681.21465968,
	"eval_logps/chosen": -25.399662037794503,
	"eval_logps/rejected": -91.77921506871728,
	"eval_loss": 0.16403362154960632,
	"eval_rewards/chosen": 1.1331302183460814,
	"eval_rewards/margins": 7.06202905465171,
	"eval_rewards/rejected": -5.9288988363056285,
	"eval_runtime": 416.8935,
	"eval_samples_per_second": 1.833,
	"eval_steps_per_second": 0.916,
	"step": 1200
	},
	{
	"epoch": 1.8151623193845,
	"grad_norm": 0.6832463145256042,
	"kl": 4.084317207336426,
	"learning_rate": 4.7649356436218484e-07,
	"logits/chosen": -100126576.3902439,
	"logits/rejected": -74491890.87179486,
	"logps/chosen": -26.829598775724087,
	"logps/rejected": -91.56940479767628,
	"loss": 0.20048983097076417,
	"rewards/chosen": 1.2928543090820312,
	"rewards/margins": 7.3325772407727365,
	"rewards/rejected": -6.039722931690705,
	"step": 1210
	},
	{
	"epoch": 1.8301745167948957,
	"grad_norm": 0.6473249793052673,
	"kl": 4.0460524559021,
	"learning_rate": 4.0380111062575555e-07,
	"logits/chosen": -102995694.13953489,
	"logits/rejected": -78813273.94594595,
	"logps/chosen": -18.408095248909884,
	"logps/rejected": -83.04783836570945,
	"loss": 0.22513113021850586,
	"rewards/chosen": 1.1756272426871366,
	"rewards/margins": 6.415878113675312,
	"rewards/rejected": -5.240250870988175,
	"step": 1220
	},
	{
	"epoch": 1.8451867142052918,
	"grad_norm": 0.9530460238456726,
	"kl": 3.9541709423065186,
	"learning_rate": 3.3700826963233735e-07,
	"logits/chosen": -109047211.6043956,
	"logits/rejected": -95853879.6521739,
	"logps/chosen": -23.388164813701923,
	"logps/rejected": -93.71621659873189,
	"loss": 0.22243864536285402,
	"rewards/chosen": 1.2722159584799966,
	"rewards/margins": 7.281630410189689,
	"rewards/rejected": -6.009414451709692,
	"step": 1230
	},
	{
	"epoch": 1.8601989116156878,
	"grad_norm": 0.6990249752998352,
	"kl": 3.6704108715057373,
	"learning_rate": 2.761561047134009e-07,
	"logits/chosen": -101100076.24691358,
	"logits/rejected": -93491297.21518987,
	"logps/chosen": -31.321445794753085,
	"logps/rejected": -85.77536590189874,
	"loss": 0.2213526725769043,
	"rewards/chosen": 1.1806782851984472,
	"rewards/margins": 6.7673752479654565,
	"rewards/rejected": -5.586696962767009,
	"step": 1240
	},
	{
	"epoch": 1.8752111090260837,
	"grad_norm": 0.6529566645622253,
	"kl": 3.52073335647583,
	"learning_rate": 2.2128202695345324e-07,
	"logits/chosen": -108384976.5925926,
	"logits/rejected": -110889893.26582278,
	"logps/chosen": -22.77246997974537,
	"logps/rejected": -85.0575059335443,
	"loss": 0.20152571201324462,
	"rewards/chosen": 1.2122876202618633,
	"rewards/margins": 6.816732293051618,
	"rewards/rejected": -5.6044446727897546,
	"step": 1250
	},
	{
	"epoch": 1.8752111090260837,
	"eval_kl": 0.12627530097961426,
	"eval_logits/chosen": -56267920.7539267,
	"eval_logits/rejected": -40249947.14136126,
	"eval_logps/chosen": -25.39533653059555,
	"eval_logps/rejected": -91.83618291884817,
	"eval_loss": 0.16398374736309052,
	"eval_rewards/chosen": 1.1335626572214497,
	"eval_rewards/margins": 7.068159173296385,
	"eval_rewards/rejected": -5.934596516074935,
	"eval_runtime": 434.1736,
	"eval_samples_per_second": 1.76,
	"eval_steps_per_second": 0.88,
	"step": 1250
	},
	{
	"epoch": 1.8902233064364795,
	"grad_norm": 0.8459712862968445,
	"kl": 3.8880774974823,
	"learning_rate": 1.7241977219020988e-07,
	"logits/chosen": -91639609.22352941,
	"logits/rejected": -86516210.34666666,
	"logps/chosen": -24.805669806985293,
	"logps/rejected": -82.68663411458333,
	"loss": 0.22068817615509034,
	"rewards/chosen": 1.1558401668772977,
	"rewards/margins": 6.285211911668965,
	"rewards/rejected": -5.1293717447916665,
	"step": 1260
	},
	{
	"epoch": 1.9052355038468756,
	"grad_norm": 0.6756543517112732,
	"kl": 3.553884983062744,
	"learning_rate": 1.29599380274259e-07,
	"logits/chosen": -110346801.09589042,
	"logits/rejected": -77342543.44827586,
	"logps/chosen": -23.693068412885275,
	"logps/rejected": -85.32204112787356,
	"loss": 0.19014220237731932,
	"rewards/chosen": 1.2435979973779965,
	"rewards/margins": 6.845515132682414,
	"rewards/rejected": -5.601917135304418,
	"step": 1270
	},
	{
	"epoch": 1.9202477012572716,
	"grad_norm": 0.47155070304870605,
	"kl": 4.422393798828125,
	"learning_rate": 9.284717660097087e-08,
	"logits/chosen": -105391794.89156626,
	"logits/rejected": -89061336.10389611,
	"logps/chosen": -21.691320947853917,
	"logps/rejected": -87.40567928165585,
	"loss": 0.2130509376525879,
	"rewards/chosen": 1.323310484369117,
	"rewards/margins": 6.870470447209012,
	"rewards/rejected": -5.547159962839895,
	"step": 1280
	},
	{
	"epoch": 1.9352598986676675,
	"grad_norm": 0.4472443163394928,
	"kl": 3.7577896118164062,
	"learning_rate": 6.218575592600085e-08,
	"logits/chosen": -110973014.36144578,
	"logits/rejected": -77297936.62337662,
	"logps/chosen": -20.669974821159638,
	"logps/rejected": -91.66918628246754,
	"loss": 0.21389095783233641,
	"rewards/chosen": 1.1915134292051017,
	"rewards/margins": 6.848199790939072,
	"rewards/rejected": -5.6566863617339695,
	"step": 1290
	},
	{
	"epoch": 1.9502720960780633,
	"grad_norm": 0.6200128197669983,
	"kl": 4.125483512878418,
	"learning_rate": 3.7633968474338756e-08,
	"logits/chosen": -104873574.4,
	"logits/rejected": -72432032.91428572,
	"logps/chosen": -22.444955783420138,
	"logps/rejected": -87.90431780133929,
	"loss": 0.22912445068359374,
	"rewards/chosen": 1.2457435607910157,
	"rewards/margins": 6.887794167654855,
	"rewards/rejected": -5.642050606863839,
	"step": 1300
	},
	{
	"epoch": 1.9502720960780633,
	"eval_kl": 0.1256106048822403,
	"eval_logits/chosen": -56246100.43979058,
	"eval_logits/rejected": -40258029.235602096,
	"eval_logps/chosen": -25.398680362401834,
	"eval_logps/rejected": -91.83032354384817,
	"eval_loss": 0.16401667892932892,
	"eval_rewards/chosen": 1.1332284817520861,
	"eval_rewards/margins": 7.067238932504704,
	"eval_rewards/rejected": -5.934010450752618,
	"eval_runtime": 423.6676,
	"eval_samples_per_second": 1.803,
	"eval_steps_per_second": 0.902,
	"step": 1300
	},
	{
	"epoch": 1.9652842934884593,
	"grad_norm": 0.702017605304718,
	"kl": 4.159964561462402,
	"learning_rate": 1.9206908351453447e-08,
	"logits/chosen": -90640182.85714285,
	"logits/rejected": -113604419.36842105,
	"logps/chosen": -23.79085431780134,
	"logps/rejected": -79.48878238075658,
	"loss": 0.22954988479614258,
	"rewards/chosen": 1.2310051690964472,
	"rewards/margins": 6.36179146252778,
	"rewards/rejected": -5.130786293431332,
	"step": 1310
	},
	{
	"epoch": 1.9802964908988554,
	"grad_norm": 0.4877679944038391,
	"kl": 4.048553943634033,
	"learning_rate": 6.915904263634643e-09,
	"logits/chosen": -131888278.58823529,
	"logits/rejected": -94457789.2173913,
	"logps/chosen": -28.130503934972428,
	"logps/rejected": -86.42369543987772,
	"loss": 0.1838226318359375,
	"rewards/chosen": 1.2597647274241728,
	"rewards/margins": 6.924890035253656,
	"rewards/rejected": -5.665125307829483,
	"step": 1320
	},
	{
	"epoch": 1.9953086883092512,
	"grad_norm": 0.5606428384780884,
	"kl": 4.262922286987305,
	"learning_rate": 7.685125532586047e-10,
	"logits/chosen": -103120477.0909091,
	"logits/rejected": -96393641.63855422,
	"logps/chosen": -19.540289544439936,
	"logps/rejected": -81.25472985692771,
	"loss": 0.21868836879730225,
	"rewards/chosen": 1.1489427244508421,
	"rewards/margins": 6.39600752172588,
	"rewards/rejected": -5.247064797275038,
	"step": 1330
	},
	{
	"epoch": 2.0,
	"eval_kl": 0.12448398768901825,
	"eval_logits/chosen": -56288711.70680628,
	"eval_logits/rejected": -40230820.85863874,
	"eval_logps/chosen": -25.394838023560208,
	"eval_logps/rejected": -91.83712369109948,
	"eval_loss": 0.16396363079547882,
	"eval_rewards/chosen": 1.1336126677028795,
	"eval_rewards/margins": 7.068303133180628,
	"eval_rewards/rejected": -5.934690465477749,
	"eval_runtime": 432.8328,
	"eval_samples_per_second": 1.765,
	"eval_steps_per_second": 0.883,
	"step": 1334
	},
	{
	"epoch": 2.0,
	"step": 1334,
	"total_flos": 0.0,
	"train_loss": 0.23600534109518803,
	"train_runtime": 42516.8613,
	"train_samples_per_second": 0.501,
	"train_steps_per_second": 0.031
	}
	],
	"logging_steps": 10,
	"max_steps": 1334,
	"num_input_tokens_seen": 0,
	"num_train_epochs": 2,
	"save_steps": 100,
	"stateful_callbacks": {
	"TrainerControl": {
	"args": {
	"should_epoch_stop": false,
	"should_evaluate": false,
	"should_log": false,
	"should_save": true,
	"should_training_stop": true
	},
	"attributes": {}
	}
	},
	"total_flos": 0.0,
	"train_batch_size": 2,
	"trial_name": null,
	"trial_params": null
	}