Upload PPO LunarLander model

README.md

@@ -0,0 +1,104 @@
+---
+tags:
+- LunarLander-v2
+- ppo
+- deep-reinforcement-learning
+- reinforcement-learning
+- custom-implementation
+model-index:
+- name: PPO
+  results:
+  - task:
+      type: reinforcement-learning
+      name: reinforcement-learning
+    dataset:
+      name: LunarLander-v2
+      type: LunarLander-v2
+    metrics:
+    - type: mean_reward
+      value: -9.92 +/- 91.50
+      name: mean_reward
+      verified: false
+---
+
+# **PPO** Agent playing **LunarLander-v2**
+
+This is a trained model of a **PPO** agent playing **LunarLander-v2** using a custom implementation.
+
+## Usage
+
+```python
+import torch
+import gymnasium as gym
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.distributions import Categorical
+import numpy as np
+
+# Define the Actor network
+class Actor(nn.Module):
+    def __init__(self, state_dim, action_dim, hidden_size=64):
+        super().__init__()
+        self.network = nn.Sequential(
+            nn.Linear(state_dim, hidden_size),
+            nn.Tanh(),
+            nn.Linear(hidden_size, hidden_size),
+            nn.Tanh(),
+            nn.Linear(hidden_size, action_dim)
+        )
+    
+    def forward(self, x):
+        return self.network(x)
+
+# Load the model
+checkpoint = torch.load("model.pt", map_location='cpu')
+actor = Actor(state_dim=8, action_dim=4, hidden_size=checkpoint['config']['hidden_size'])
+actor.load_state_dict(checkpoint['actor_state_dict'])
+actor.eval()
+
+# Test the agent
+env = gym.make("LunarLander-v2")
+state, _ = env.reset()
+total_reward = 0
+
+for _ in range(1000):  # Max steps
+    with torch.no_grad():
+        state_tensor = torch.FloatTensor(state).unsqueeze(0)
+        logits = actor(state_tensor)
+        action = torch.argmax(logits, dim=-1).item()
+    
+    state, reward, terminated, truncated, _ = env.step(action)
+    total_reward += reward
+    
+    if terminated or truncated:
+        break
+
+print(f"Total reward: {total_reward:.2f}")
+```
+
+## Training Results
+
+- **Mean reward**: -9.92 ± 91.50
+- **Best reward**: 143.33
+- **Success rate**: 0.0% (episodes with reward > 200)
+- **Total episodes**: 353
+- **Total timesteps**: 100,000
+
+## Algorithm Configuration
+
+- **Algorithm**: Proximal Policy Optimization (PPO)
+- **Learning rate**: 0.0003
+- **Batch size**: 2048
+- **Clip coefficient**: 0.2
+- **Entropy coefficient**: 0.01
+- **Value coefficient**: 0.5
+- **Gamma**: 0.99
+- **GAE Lambda**: 0.95
+
+## Training Environment
+
+- **Environment**: LunarLander-v2
+- **Framework**: PyTorch + Gymnasium
+- **Training date**: 2025-09-04
+
+This model was trained as part of the Hugging Face Deep RL Course.

config.json

@@ -0,0 +1,18 @@
+{
+  "env_id": "LunarLander-v2",
+  "max_episode_steps": 1000,
+  "total_timesteps": 100000,
+  "batch_size": 2048,
+  "minibatch_size": 64,
+  "num_epochs": 4,
+  "learning_rate": 0.0003,
+  "gamma": 0.99,
+  "gae_lambda": 0.95,
+  "clip_coef": 0.2,
+  "entropy_coef": 0.01,
+  "value_coef": 0.5,
+  "max_grad_norm": 0.5,
+  "hidden_size": 64,
+  "log_frequency": 1000,
+  "eval_frequency": 5000
+}

model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2984c79fe12f2cffd8ed7a990bf3ca40532c544591abdd38b7c1fee9f74f5413
+size 65115

requirements.txt

@@ -0,0 +1,4 @@
+torch>=1.9.0
+gymnasium[box2d]>=0.28.0
+numpy>=1.21.0
+matplotlib>=3.3.0

results.json

@@ -0,0 +1,866 @@
+{
+  "episode_rewards": [
+    -0.06825948606616805,
+    -169.11205875315596,
+    -215.3913619178971,
+    -139.8789067666994,
+    -509.00763208625216,
+    -109.10158643440585,
+    -166.36546227855413,
+    -200.2774249561457,
+    -177.853047471592,
+    -117.68830352738027,
+    -377.8491467246267,
+    -88.45872897085687,
+    -87.81601692565125,
+    -95.75884386471762,
+    -42.98813630908335,
+    -243.49287632618473,
+    -139.87800064187832,
+    -104.03428716974196,
+    -390.8048731990434,
+    -225.5430582633193,
+    -185.89923995942115,
+    -88.42383149105987,
+    -408.9225997545925,
+    -106.22383469588478,
+    -141.10328573561222,
+    -87.93442530313699,
+    -79.96148810811498,
+    -132.9337179443088,
+    -213.97733893091453,
+    -147.42151069764924,
+    -162.99907861266115,
+    -144.6632461507486,
+    -84.81459503872658,
+    -69.14377921641031,
+    -104.60268858246573,
+    -57.19420918932683,
+    -161.42868563828253,
+    -344.4091094104822,
+    -101.16971336992118,
+    -188.49448757234114,
+    -13.678164633553635,
+    -108.50645100939873,
+    -170.76606043027135,
+    -186.7262446158587,
+    -15.216783348278511,
+    -114.86771345279944,
+    -227.3692629653085,
+    -147.20910772391082,
+    -129.51296531717037,
+    -169.92751893892438,
+    -92.67419463241187,
+    -66.52312287592397,
+    -205.00254460937958,
+    -294.4399920099744,
+    -68.70825604829983,
+    -95.73002166672707,
+    -203.00938076883114,
+    -180.01476651566466,
+    -218.65008045854182,
+    -109.9569345200619,
+    -249.13048322194444,
+    -66.71251085050328,
+    -96.39204623549715,
+    -115.68446613589023,
+    -163.87003732668143,
+    -177.90045049613406,
+    -75.71805993225172,
+    -123.02066775256463,
+    -73.64389095234229,
+    -261.12391153130307,
+    -106.47186764493826,
+    -101.80564931343925,
+    -141.86797230927073,
+    -156.61153573618833,
+    -95.2393424830547,
+    -90.22571965883145,
+    -143.6636948877111,
+    -51.35073548228167,
+    -519.0832165573745,
+    -341.6084146927546,
+    -79.28020255548074,
+    -119.36101244775585,
+    -88.1950180493556,
+    -122.61044158853123,
+    -135.71958328905532,
+    -207.58081741472392,
+    -144.75407671414126,
+    -83.38352184054375,
+    -176.40191781750895,
+    -101.60612676644908,
+    -75.21788949050259,
+    -315.1279736505058,
+    -207.51669932911682,
+    -48.31907149982139,
+    -308.26873149443145,
+    -219.67469874966247,
+    -137.19352252669634,
+    -44.806359347531014,
+    -107.76555172357948,
+    -297.23318223874264,
+    -186.3685812608377,
+    -110.65739435770014,
+    -253.53768551432907,
+    -178.73459600772205,
+    -86.00970854249826,
+    -71.44504434241344,
+    -315.7744485503431,
+    -94.420414320224,
+    -207.2584697830588,
+    -71.94779060839778,
+    -135.1856849970423,
+    -55.45307758594119,
+    -137.02197706792916,
+    -173.33883858632197,
+    -116.13620467096102,
+    -100.33774882902676,
+    -96.21422856581302,
+    -306.7805674821406,
+    -115.20336496789074,
+    -91.53036352443942,
+    -124.2550546719635,
+    -44.74700688984777,
+    -74.32923924165786,
+    -106.72254156258597,
+    -145.1791552903415,
+    -325.6934198758087,
+    -492.27073137336924,
+    -86.99830724178076,
+    -175.6210715671713,
+    -331.7468607743321,
+    -85.70705823312146,
+    -105.0045904423163,
+    -99.45017495028988,
+    -78.43215638806684,
+    -161.86971764719186,
+    -99.54601422371243,
+    -117.09788268141342,
+    -247.03344075957054,
+    -165.6021271585296,
+    -134.35843308237764,
+    -49.73205798092371,
+    23.212409800418598,
+    -115.15950679935591,
+    -80.7186329423706,
+    -63.94584994249098,
+    -94.55277414944797,
+    -85.40629043215664,
+    -50.799756615195946,
+    -87.72580112853257,
+    -135.9770036205606,
+    -138.92725370070528,
+    -76.66805661003464,
+    -117.18797733566853,
+    -128.15344308974312,
+    -167.25237561311252,
+    -180.84063140488854,
+    -93.59998844842957,
+    -191.19317336997725,
+    -26.29411332936955,
+    -96.49525715689076,
+    -145.29278234776177,
+    -119.57008506165374,
+    -133.13342043800822,
+    -134.65198237996353,
+    -76.58624460994515,
+    -70.34120337383676,
+    -59.63430906498967,
+    -357.6935432146067,
+    -179.39487446119162,
+    -337.56760565449656,
+    -49.05309353091758,
+    -152.00949803010212,
+    -67.1993251644055,
+    -6.099140936544586,
+    -14.201823869525427,
+    -81.7722795475811,
+    -200.1448172618197,
+    -184.37469314203383,
+    -36.92118377875647,
+    -90.34611709777525,
+    -68.6805904834934,
+    -53.716997900380434,
+    -23.113558688573406,
+    -8.903564346705423,
+    -27.55353003701832,
+    -55.428221828307414,
+    -84.96989184224451,
+    -115.05392030264886,
+    -53.77243026924064,
+    -125.56693025600609,
+    -127.89382603352178,
+    -298.05774179855473,
+    -59.40924420265998,
+    14.639449861942666,
+    -125.90139888201305,
+    -28.511688106958374,
+    -227.56883861154222,
+    -226.9705435777895,
+    -233.8241736362481,
+    -145.25056610537473,
+    -145.1836740137075,
+    -380.3525086402928,
+    -439.9677258270466,
+    -211.5204267450631,
+    0.11140423555376344,
+    -173.3141272144474,
+    -369.91312641847395,
+    -247.012128973268,
+    -2.5561605861201144,
+    -88.12799381680097,
+    -84.81944076821246,
+    -112.23794903888829,
+    -134.88684454749006,
+    -46.5822621434159,
+    -37.89523863193384,
+    -55.66770982032418,
+    -192.16655727616046,
+    -221.83124282901133,
+    -69.30029113382633,
+    -63.65819587905243,
+    -159.41511505490848,
+    -268.658235468461,
+    1.5979156618273151,
+    -132.945999852996,
+    -2.5585262389231644,
+    -229.86666718444704,
+    -33.50987358508034,
+    -91.74437579021038,
+    -13.499084717157515,
+    -33.91904110843143,
+    83.74854941453819,
+    14.740936458834327,
+    -17.505239573516818,
+    17.005275529258597,
+    -64.13899263345624,
+    -1.8494117922500948,
+    -13.80694293885719,
+    -161.59203053864863,
+    -5.111877413591586,
+    -62.267203761515994,
+    -203.1636986569103,
+    -16.33521283772599,
+    -43.76419289703527,
+    -231.79356791613571,
+    -19.82721510771465,
+    -24.94162644332951,
+    -140.13489191875996,
+    -35.33350810426582,
+    -25.517750232653157,
+    -323.82197700896745,
+    -33.1830087723218,
+    -58.84664699201487,
+    -53.8204562164564,
+    -94.41007698626146,
+    20.1768324848328,
+    39.568090697977766,
+    28.742513615873946,
+    -10.876059937404719,
+    36.887560542335905,
+    57.72817332988835,
+    -121.2429640394513,
+    -277.64083651135974,
+    36.56318243984942,
+    -189.75141689003476,
+    -102.57304618954247,
+    -25.353143435251624,
+    -81.33489379005088,
+    -34.462574286471536,
+    -13.184893660098453,
+    -20.9300185028616,
+    -43.849572287832835,
+    -61.405688917778264,
+    4.850469385603915,
+    73.42787441544748,
+    -14.142018323762258,
+    -10.218390830410101,
+    -42.09291614967388,
+    6.446253695358948,
+    -89.34496052487404,
+    7.730260234043357,
+    -3.2801650797843678,
+    40.05943458622408,
+    3.4305993288540044,
+    -181.9884743359073,
+    -48.15835818232749,
+    -62.52116058763089,
+    -69.72306275970666,
+    -36.67234925154844,
+    -2.576860974653428,
+    74.94141121003486,
+    7.110245109088272,
+    -121.51373599032131,
+    41.62826356222841,
+    32.69544163530229,
+    -10.915920698598285,
+    38.00125649565851,
+    -127.1549093744554,
+    45.17158197251346,
+    -291.45632312199194,
+    -347.4260170142368,
+    -177.93849720833748,
+    -151.55347718264545,
+    -187.41104423710306,
+    -24.44516627724018,
+    -92.84446205697854,
+    -68.53762724577287,
+    -117.39158029036854,
+    6.951101977978475,
+    -111.45460200674995,
+    37.59049691017192,
+    76.11035439251663,
+    60.072023854697335,
+    132.17751182796914,
+    95.66148791008895,
+    127.01426732947682,
+    -28.125136920530366,
+    80.49117019603592,
+    99.07148572522087,
+    80.90938719489925,
+    -12.548944087098235,
+    91.4527720834068,
+    10.987130469205631,
+    63.18406103139541,
+    -35.32557212165236,
+    -12.361553198005272,
+    85.70227644691255,
+    109.80887623490239,
+    60.50440531795487,
+    50.73971224683154,
+    -92.68465262159447,
+    -19.7173009646866,
+    49.82634045083176,
+    34.626965702974054,
+    9.144020124847328,
+    84.83352703661873,
+    30.00431252390878,
+    57.90072859438916,
+    85.08031680993007,
+    -51.65908002767094,
+    -39.021653890489404,
+    -44.40585873624194,
+    60.18370016305159,
+    87.8427834734688,
+    85.46914135213747,
+    91.95126816001013,
+    -71.52945051996913,
+    -92.6781186850232,
+    51.24324737252195,
+    87.4071023140425,
+    143.33163700976513,
+    59.59649815264238,
+    67.47989524602491,
+    26.09265425121593
+  ],
+  "episode_lengths": [
+    99,
+    109,
+    113,
+    120,
+    87,
+    107,
+    74,
+    108,
+    88,
+    82,
+    90,
+    158,
+    74,
+    66,
+    99,
+    113,
+    91,
+    79,
+    118,
+    85,
+    89,
+    77,
+    103,
+    96,
+    109,
+    77,
+    105,
+    73,
+    141,
+    128,
+    66,
+    122,
+    95,
+    75,
+    82,
+    86,
+    118,
+    121,
+    110,
+    89,
+    89,
+    94,
+    70,
+    100,
+    74,
+    106,
+    111,
+    71,
+    116,
+    105,
+    66,
+    68,
+    93,
+    125,
+    70,
+    89,
+    102,
+    89,
+    125,
+    78,
+    113,
+    97,
+    80,
+    71,
+    154,
+    93,
+    66,
+    113,
+    93,
+    106,
+    94,
+    129,
+    70,
+    74,
+    83,
+    75,
+    83,
+    68,
+    113,
+    133,
+    68,
+    69,
+    66,
+    82,
+    92,
+    144,
+    101,
+    72,
+    99,
+    129,
+    72,
+    133,
+    127,
+    75,
+    136,
+    113,
+    77,
+    106,
+    106,
+    148,
+    111,
+    71,
+    131,
+    124,
+    72,
+    139,
+    120,
+    132,
+    142,
+    72,
+    88,
+    153,
+    134,
+    108,
+    91,
+    192,
+    81,
+    146,
+    84,
+    79,
+    97,
+    159,
+    103,
+    101,
+    133,
+    174,
+    177,
+    134,
+    101,
+    137,
+    77,
+    87,
+    79,
+    141,
+    122,
+    97,
+    85,
+    158,
+    117,
+    110,
+    173,
+    108,
+    77,
+    100,
+    72,
+    96,
+    77,
+    73,
+    72,
+    109,
+    178,
+    107,
+    116,
+    94,
+    112,
+    135,
+    151,
+    171,
+    92,
+    131,
+    103,
+    157,
+    163,
+    94,
+    71,
+    116,
+    116,
+    234,
+    100,
+    130,
+    162,
+    202,
+    142,
+    142,
+    102,
+    99,
+    206,
+    132,
+    175,
+    159,
+    98,
+    170,
+    98,
+    97,
+    82,
+    172,
+    157,
+    130,
+    129,
+    222,
+    245,
+    107,
+    116,
+    153,
+    95,
+    76,
+    178,
+    158,
+    186,
+    185,
+    254,
+    313,
+    221,
+    147,
+    177,
+    125,
+    227,
+    203,
+    124,
+    275,
+    111,
+    182,
+    281,
+    215,
+    154,
+    109,
+    233,
+    192,
+    127,
+    211,
+    324,
+    206,
+    112,
+    139,
+    203,
+    226,
+    131,
+    126,
+    159,
+    115,
+    1000,
+    250,
+    192,
+    174,
+    94,
+    90,
+    139,
+    199,
+    137,
+    119,
+    148,
+    120,
+    110,
+    325,
+    173,
+    274,
+    304,
+    281,
+    225,
+    211,
+    222,
+    219,
+    225,
+    343,
+    191,
+    200,
+    137,
+    170,
+    143,
+    1000,
+    452,
+    260,
+    1000,
+    273,
+    212,
+    208,
+    220,
+    210,
+    132,
+    178,
+    185,
+    243,
+    348,
+    1000,
+    205,
+    322,
+    136,
+    1000,
+    208,
+    223,
+    211,
+    1000,
+    1000,
+    205,
+    194,
+    215,
+    376,
+    334,
+    363,
+    1000,
+    1000,
+    656,
+    1000,
+    1000,
+    257,
+    1000,
+    702,
+    1000,
+    573,
+    732,
+    330,
+    1000,
+    314,
+    1000,
+    445,
+    523,
+    377,
+    1000,
+    690,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    479,
+    1000,
+    1000,
+    1000,
+    308,
+    1000,
+    1000,
+    1000,
+    289,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    674,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    568,
+    336,
+    510,
+    1000,
+    1000,
+    1000,
+    1000,
+    426,
+    450,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000,
+    1000
+  ],
+  "training_losses": {
+    "actor": [
+      -0.01979029180802172,
+      -0.01952682618139079,
+      -0.023040959014906548,
+      -0.021843695227289572,
+      -0.01714207920304034,
+      -0.014676372746180277,
+      -0.017646560751018114,
+      -0.0215543580125086,
+      -0.014346814270538744,
+      -0.021339075115974993,
+      -0.016831182496389374,
+      -0.022772795979108196,
+      -0.023433746107912157,
+      -0.018633067615155596,
+      -0.01597626227157889,
+      -0.01767163053591503,
+      -0.01928800484893145,
+      -0.02197402838646667,
+      -0.018846331498934887,
+      -0.018185104410804342,
+      -0.017785934993298724,
+      -0.017179835092974827,
+      -0.018634800864674617,
+      -0.016147430505952798,
+      -0.020907533988065552,
+      -0.014310308593849186,
+      -0.01923305022501154,
+      -0.018182744635851122,
+      -0.016129654854012188,
+      -0.018532538808358368,
+      -0.015569949602650013,
+      -0.017233232538274024,
+      -0.01921561570270569,
+      -0.014850864667096175,
+      -0.013190674544603098,
+      -0.011923887774173636,
+      -0.017301808853517286,
+      -0.01568447455065325,
+      -0.012880522626801394,
+      -0.016250163862423506,
+      -0.01898401835205732,
+      -0.016680118802469224,
+      -0.015046855827677064,
+      -0.013486761541571468,
+      -0.014262968943512533,
+      -0.012893502258521039,
+      -0.014304315525805578,
+      -0.012067950345226564
+    ],
+    "critic": [
+      1680.4626302719116,
+      1030.951669216156,
+      1021.7877514362335,
+      1272.4353795051575,
+      1158.8028721809387,
+      727.7568366527557,
+      901.111081123352,
+      543.9186744689941,
+      530.4165495634079,
+      489.02578616142273,
+      528.2152299880981,
+      495.15413546562195,
+      269.65131109952927,
+      315.82098948955536,
+      372.054971575737,
+      369.43128794431686,
+      209.46067798137665,
+      245.78451192378998,
+      218.96768248081207,
+      120.53185752034187,
+      160.47752958536148,
+      162.42764976620674,
+      156.59158584475517,
+      67.61967559158802,
+      56.876462534070015,
+      80.85796889662743,
+      218.38137596845627,
+      178.75969290733337,
+      131.04233753681183,
+      98.44907432794571,
+      30.182169884443283,
+      46.721587389707565,
+      103.00322636961937,
+      28.124917030334473,
+      90.61402375996113,
+      33.96754629909992,
+      53.8245303183794,
+      24.37088042497635,
+      115.26408568024635,
+      29.321963973343372,
+      26.280027896165848,
+      19.028836239129305,
+      15.724595122039318,
+      168.65730379521847,
+      17.816705368459225,
+      117.39989982545376,
+      25.387532763183117,
+      7.2297560181468725
+    ],
+    "entropy": [
+      1.384157688356936,
+      1.3705531377345324,
+      1.3620696673169732,
+      1.34917978849262,
+      1.33616404235363,
+      1.333935335278511,
+      1.3117941915988922,
+      1.2774329232051969,
+      1.2390660690143704,
+      1.2097994657233357,
+      1.1650842120870948,
+      1.163279932923615,
+      1.1738641979172826,
+      1.2158568240702152,
+      1.114059129729867,
+      1.124722182750702,
+      1.129445598460734,
+      1.2377208853140473,
+      1.1542788790538907,
+      1.2222738303244114,
+      1.1691155284643173,
+      1.165976474992931,
+      1.1471045780926943,
+      1.1464839773252606,
+      1.1270105578005314,
+      0.9089711038395762,
+      1.1209891475737095,
+      1.0858190059661865,
+      1.0796098113059998,
+      1.1417838828638196,
+      1.1433162745088339,
+      1.0739809516817331,
+      1.085077359341085,
+      1.0622714115306735,
+      1.0335699897259474,
+      1.0388436089269817,
+      1.064063385128975,
+      1.034223263617605,
+      1.0418634568341076,
+      0.9904597336426377,
+      0.895952848251909,
+      1.0142980953678489,
+      1.0475381733849645,
+      0.9823213117197156,
+      0.9869043030776083,
+      0.9921243949793279,
+      0.8546506622806191,
+      0.9553067521192133
+    ]
+  },
+  "total_timesteps": 100000,
+  "total_episodes": 353
+}

test_model.py

@@ -0,0 +1,47 @@
+import torch
+import gymnasium as gym
+import torch.nn as nn
+
+class Actor(nn.Module):
+    def __init__(self, state_dim, action_dim, hidden_size=64):
+        super().__init__()
+        self.network = nn.Sequential(
+            nn.Linear(state_dim, hidden_size),
+            nn.Tanh(),
+            nn.Linear(hidden_size, hidden_size),
+            nn.Tanh(),
+            nn.Linear(hidden_size, action_dim)
+        )
+    
+    def forward(self, x):
+        return self.network(x)
+
+def test_model():
+    # Load the model
+    checkpoint = torch.load("model.pt", map_location='cpu')
+    actor = Actor(state_dim=8, action_dim=4, hidden_size=checkpoint['config']['hidden_size'])
+    actor.load_state_dict(checkpoint['actor_state_dict'])
+    actor.eval()
+    
+    # Test the agent
+    env = gym.make("LunarLander-v2", render_mode="human")
+    state, _ = env.reset()
+    total_reward = 0
+    
+    for _ in range(1000):
+        with torch.no_grad():
+            state_tensor = torch.FloatTensor(state).unsqueeze(0)
+            logits = actor(state_tensor)
+            action = torch.argmax(logits, dim=-1).item()
+        
+        state, reward, terminated, truncated, _ = env.step(action)
+        total_reward += reward
+        
+        if terminated or truncated:
+            break
+    
+    env.close()
+    print(f"Total reward: {total_reward:.2f}")
+
+if __name__ == "__main__":
+    test_model()