Hugging Face's logo

OneScience-Group
/
ClayFoundation

English
clayfoundation
OneScience
Earth Science
Multi-Source Earth Observation
Remote Sensing Foundation Model
Model card Files
xet
 Community
ClayFoundation
File size: 2,913 Bytes
53becf5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
{
  "model_name": "ClayFoundation",
  "model_type": "clayfoundation",
  "architectures": [
    "ClayFoundation",
    "DynamicEmbedding",
    "Transformer",
    "DynamicDecoder"
  ],
  "framework": "PyTorch",
  "domain": "earth-observation",
  "task": "multi-sensor-remote-sensing-representation-learning",
  "implementation": {
    "entry_point": "model/clayfoundation.py",
    "scope": "scaled-down independent engineering reproduction of the Clay v1.5 multi-sensor masked-autoencoder specification",
    "train_script": "scripts/train.py",
    "inference_script": "scripts/inference.py",
    "evaluation_script": "scripts/result.py",
    "synthetic_data_script": "scripts/fake_data.py"
  },
  "architecture": {
    "family": "multi-sensor masked autoencoder with wavelength-conditioned dynamic embeddings",
    "patch_size": 8,
    "mask_ratio": 0.75,
    "encoder_dim": 96,
    "encoder_depth": 2,
    "encoder_heads": 4,
    "decoder_dim": 64,
    "decoder_depth": 1,
    "decoder_heads": 4,
    "wave_dim": 32,
    "wave_latents": 16,
    "teacher_dim": 128,
    "norm_pix_loss": false,
    "metadata": [
      "week",
      "hour",
      "latitude",
      "longitude",
      "ground_sample_distance",
      "band_center_wavelengths"
    ],
    "paper_model": {
      "patch_size": 8,
      "mask_ratio": 0.75,
      "encoder_dim": 1024,
      "encoder_depth": 24,
      "encoder_heads": 16,
      "decoder_dim": 512,
      "decoder_depth": 4,
      "decoder_heads": 4,
      "parameter_count": "632M including DINOv2 teacher",
      "encoder_parameter_count": "311M",
      "teacher": "DINOv2 304M"
    }
  },
  "data": {
    "datasets": [
      "Synthetic Multi-Sensor Earth Observation Chips"
    ],
    "protocol": "clay_synthetic_v1_5_engineering_v1",
    "format": "NPZ",
    "train_file": "data/train.npz",
    "test_file": "data/test.npz",
    "image_size": 64,
    "train_samples": 4,
    "test_samples": 2,
    "sensors": {
      "sentinel-2-l2a": {
        "channels": 10,
        "gsd_m": 10.0,
        "wavelengths_nm": [493, 560, 665, 704, 740, 783, 842, 865, 1610, 2190],
        "rgb_indices": [2, 1, 0]
      },
      "landsat-c2l2-sr": {
        "channels": 6,
        "gsd_m": 30.0,
        "wavelengths_nm": [650, 560, 480, 860, 1600, 2200],
        "rgb_indices": [0, 1, 2]
      },
      "sentinel-1-rtc": {
        "channels": 2,
        "gsd_m": 10.0,
        "wavelengths_nm": [3500, 4000],
        "rgb_indices": [0, 1, 0]
      }
    },
    "required_metadata": [
      "format_version",
      "data_source",
      "time",
      "latlon",
      "class_target",
      "regression_target",
      "teacher_target",
      "pixels",
      "valid",
      "wavelengths",
      "gsd"
    ]
  },
  "configuration_sources": [
    "conf/config.yaml",
    "model/clayfoundation.py",
    "scripts/fake_data.py",
    "scripts/train.py",
    "scripts/inference.py",
    "scripts/result.py"
  ]
}