yzzhao commited on
Commit
fa91007
·
verified ·
1 Parent(s): 48e3a01

Update native Kinex Hard results: 9 completed valid episodes

Browse files
data-source.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "schema": "programvla/dataset-source/1",
3
  "dataset": "RLE-Bench/programmable-vla",
4
- "revision": "e943c539ee9205c818107b58a4707734de87710a",
5
- "evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/e943c539ee9205c818107b58a4707734de87710a/",
6
  "files": {
7
  "data.json": {
8
  "url": "./results/data.json"
 
1
  {
2
  "schema": "programvla/dataset-source/1",
3
  "dataset": "RLE-Bench/programmable-vla",
4
+ "revision": "7c93400e87c06e4f0bcd422c21a965e74893f361",
5
+ "evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/7c93400e87c06e4f0bcd422c21a965e74893f361/",
6
  "files": {
7
  "data.json": {
8
  "url": "./results/data.json"
results/cost-data.json CHANGED
@@ -7925,6 +7925,238 @@
7925
  "long_context_responses": 0,
7926
  "cache_write_assumed": false,
7927
  "evidence_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/session.jsonl"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7928
  }
7929
  },
7930
  "sources": [],
 
7925
  "long_context_responses": 0,
7926
  "cache_write_assumed": false,
7927
  "evidence_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/session.jsonl"
7928
+ },
7929
+ "native-kinex-01-20261113": {
7930
+ "signature": [
7931
+ "native-kinex-01-20261113",
7932
+ "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/usage.json",
7933
+ 2555920,
7934
+ 11275,
7935
+ 2502272,
7936
+ 0,
7937
+ null
7938
+ ],
7939
+ "lower": 3.6025020000000003,
7940
+ "upper": 3.6025020000000003,
7941
+ "actual_billed_usd": null,
7942
+ "basis": "native_pi_session_api_equivalent",
7943
+ "buckets": [
7944
+ {
7945
+ "category": "fresh_input",
7946
+ "tier": "Recorded session",
7947
+ "tokens": 53648,
7948
+ "rate": 10.0,
7949
+ "rate_upper": 10.0,
7950
+ "lower": 0.5364800000000001,
7951
+ "upper": 0.5364800000000001
7952
+ },
7953
+ {
7954
+ "category": "cache_read",
7955
+ "tier": "Recorded session",
7956
+ "tokens": 2502272,
7957
+ "rate": 1.0,
7958
+ "rate_upper": 1.0,
7959
+ "lower": 2.502272,
7960
+ "upper": 2.502272
7961
+ },
7962
+ {
7963
+ "category": "cache_write",
7964
+ "tier": "Recorded session",
7965
+ "tokens": 0,
7966
+ "rate": null,
7967
+ "rate_upper": null,
7968
+ "lower": 0,
7969
+ "upper": 0
7970
+ },
7971
+ {
7972
+ "category": "output",
7973
+ "tier": "Recorded session",
7974
+ "tokens": 11275,
7975
+ "rate": 50.00000000000001,
7976
+ "rate_upper": 50.00000000000001,
7977
+ "lower": 0.5637500000000001,
7978
+ "upper": 0.5637500000000001
7979
+ }
7980
+ ],
7981
+ "cache_lower": 2502272,
7982
+ "cache_upper": 2502272,
7983
+ "long_context_responses": 0,
7984
+ "cache_write_assumed": false,
7985
+ "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/session.jsonl"
7986
+ },
7987
+ "native-kinex-03-20261113": {
7988
+ "signature": [
7989
+ "native-kinex-03-20261113",
7990
+ "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/usage.json",
7991
+ 2014191,
7992
+ 9624,
7993
+ 1972480,
7994
+ 0,
7995
+ null
7996
+ ],
7997
+ "lower": 2.87079,
7998
+ "upper": 2.87079,
7999
+ "actual_billed_usd": null,
8000
+ "basis": "native_pi_session_api_equivalent",
8001
+ "buckets": [
8002
+ {
8003
+ "category": "fresh_input",
8004
+ "tier": "Recorded session",
8005
+ "tokens": 41711,
8006
+ "rate": 10.0,
8007
+ "rate_upper": 10.0,
8008
+ "lower": 0.41711000000000004,
8009
+ "upper": 0.41711000000000004
8010
+ },
8011
+ {
8012
+ "category": "cache_read",
8013
+ "tier": "Recorded session",
8014
+ "tokens": 1972480,
8015
+ "rate": 1.0,
8016
+ "rate_upper": 1.0,
8017
+ "lower": 1.97248,
8018
+ "upper": 1.97248
8019
+ },
8020
+ {
8021
+ "category": "cache_write",
8022
+ "tier": "Recorded session",
8023
+ "tokens": 0,
8024
+ "rate": null,
8025
+ "rate_upper": null,
8026
+ "lower": 0,
8027
+ "upper": 0
8028
+ },
8029
+ {
8030
+ "category": "output",
8031
+ "tier": "Recorded session",
8032
+ "tokens": 9624,
8033
+ "rate": 50.0,
8034
+ "rate_upper": 50.0,
8035
+ "lower": 0.4812,
8036
+ "upper": 0.4812
8037
+ }
8038
+ ],
8039
+ "cache_lower": 1972480,
8040
+ "cache_upper": 1972480,
8041
+ "long_context_responses": 0,
8042
+ "cache_write_assumed": false,
8043
+ "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/session.jsonl"
8044
+ },
8045
+ "native-kinex-05-20261113": {
8046
+ "signature": [
8047
+ "native-kinex-05-20261113",
8048
+ "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/usage.json",
8049
+ 2051199,
8050
+ 9278,
8051
+ 2002432,
8052
+ 0,
8053
+ null
8054
+ ],
8055
+ "lower": 2.954002,
8056
+ "upper": 2.954002,
8057
+ "actual_billed_usd": null,
8058
+ "basis": "native_pi_session_api_equivalent",
8059
+ "buckets": [
8060
+ {
8061
+ "category": "fresh_input",
8062
+ "tier": "Recorded session",
8063
+ "tokens": 48767,
8064
+ "rate": 10.0,
8065
+ "rate_upper": 10.0,
8066
+ "lower": 0.48767000000000005,
8067
+ "upper": 0.48767000000000005
8068
+ },
8069
+ {
8070
+ "category": "cache_read",
8071
+ "tier": "Recorded session",
8072
+ "tokens": 2002432,
8073
+ "rate": 1.0,
8074
+ "rate_upper": 1.0,
8075
+ "lower": 2.0024319999999998,
8076
+ "upper": 2.0024319999999998
8077
+ },
8078
+ {
8079
+ "category": "cache_write",
8080
+ "tier": "Recorded session",
8081
+ "tokens": 0,
8082
+ "rate": null,
8083
+ "rate_upper": null,
8084
+ "lower": 0,
8085
+ "upper": 0
8086
+ },
8087
+ {
8088
+ "category": "output",
8089
+ "tier": "Recorded session",
8090
+ "tokens": 9278,
8091
+ "rate": 50.0,
8092
+ "rate_upper": 50.0,
8093
+ "lower": 0.46390000000000003,
8094
+ "upper": 0.46390000000000003
8095
+ }
8096
+ ],
8097
+ "cache_lower": 2002432,
8098
+ "cache_upper": 2002432,
8099
+ "long_context_responses": 0,
8100
+ "cache_write_assumed": false,
8101
+ "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/session.jsonl"
8102
+ },
8103
+ "native-kinex-vlact50k-10-20261113": {
8104
+ "signature": [
8105
+ "native-kinex-vlact50k-10-20261113",
8106
+ "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/usage.json",
8107
+ 946198,
8108
+ 4960,
8109
+ 911744,
8110
+ 0,
8111
+ null
8112
+ ],
8113
+ "lower": 1.504284,
8114
+ "upper": 1.504284,
8115
+ "actual_billed_usd": null,
8116
+ "basis": "native_pi_session_api_equivalent",
8117
+ "buckets": [
8118
+ {
8119
+ "category": "fresh_input",
8120
+ "tier": "Recorded session",
8121
+ "tokens": 34454,
8122
+ "rate": 10.0,
8123
+ "rate_upper": 10.0,
8124
+ "lower": 0.34454,
8125
+ "upper": 0.34454
8126
+ },
8127
+ {
8128
+ "category": "cache_read",
8129
+ "tier": "Recorded session",
8130
+ "tokens": 911744,
8131
+ "rate": 1.0,
8132
+ "rate_upper": 1.0,
8133
+ "lower": 0.911744,
8134
+ "upper": 0.911744
8135
+ },
8136
+ {
8137
+ "category": "cache_write",
8138
+ "tier": "Recorded session",
8139
+ "tokens": 0,
8140
+ "rate": null,
8141
+ "rate_upper": null,
8142
+ "lower": 0,
8143
+ "upper": 0
8144
+ },
8145
+ {
8146
+ "category": "output",
8147
+ "tier": "Recorded session",
8148
+ "tokens": 4960,
8149
+ "rate": 50.0,
8150
+ "rate_upper": 50.0,
8151
+ "lower": 0.24800000000000003,
8152
+ "upper": 0.24800000000000003
8153
+ }
8154
+ ],
8155
+ "cache_lower": 911744,
8156
+ "cache_upper": 911744,
8157
+ "long_context_responses": 0,
8158
+ "cache_write_assumed": false,
8159
+ "evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/session.jsonl"
8160
  }
8161
  },
8162
  "sources": [],
results/data.json CHANGED
The diff for this file is too large to render. See raw diff
 
results/measured-instructions.json CHANGED
@@ -1941,6 +1941,50 @@
1941
  "instructions_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/instruction.json",
1942
  "transcript_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/transcript.json"
1943
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1944
  }
1945
  }
1946
  }
 
1941
  "instructions_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/instruction.json",
1942
  "transcript_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/transcript.json"
1943
  }
1944
+ },
1945
+ "native-kinex-01-20261113": {
1946
+ "task_key": "task03/01",
1947
+ "initial_goal": "Grab the red block with the left arm, transfer it to the right arm, and set it on the blue pad.",
1948
+ "text": "Grab the red block with the left arm, transfer it to the right arm, and set it on the blue pad.",
1949
+ "kind": "task_instruction",
1950
+ "observed_native": "Grab the red block with the left arm, transfer it to the right arm, and set it on the blue pad.",
1951
+ "source": {
1952
+ "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/instruction.json",
1953
+ "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/transcript.json"
1954
+ }
1955
+ },
1956
+ "native-kinex-03-20261113": {
1957
+ "task_key": "task03/03",
1958
+ "initial_goal": "Place the yellow bottle with elongated cylinder body, the bottle with blue highlights, and the bottle with red screw cap in the dustbin the simple black garbage bin.",
1959
+ "text": "Place the yellow bottle with elongated cylinder body, the bottle with blue highlights, and the bottle with red screw cap in the dustbin the simple black garbage bin.",
1960
+ "kind": "task_instruction",
1961
+ "observed_native": "Place the yellow bottle with elongated cylinder body, the bottle with blue highlights, and the bottle with red screw cap in the dustbin the simple black garbage bin.",
1962
+ "source": {
1963
+ "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/instruction.json",
1964
+ "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/transcript.json"
1965
+ }
1966
+ },
1967
+ "native-kinex-05-20261113": {
1968
+ "task_key": "task03/05",
1969
+ "initial_goal": "Grab the smooth bowl with glossy finish, stack them one by one.",
1970
+ "text": "Grab the smooth bowl with glossy finish, stack them one by one.",
1971
+ "kind": "task_instruction",
1972
+ "observed_native": "Grab the smooth bowl with glossy finish, stack them one by one.",
1973
+ "source": {
1974
+ "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/instruction.json",
1975
+ "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/transcript.json"
1976
+ }
1977
+ },
1978
+ "native-kinex-vlact50k-10-20261113": {
1979
+ "task_key": "task03/10",
1980
+ "initial_goal": "Hold the dark gray cooking kitchenpot firmly and lift it",
1981
+ "text": "Hold the dark gray cooking kitchenpot firmly and lift it",
1982
+ "kind": "task_instruction",
1983
+ "observed_native": "Hold the dark gray cooking kitchenpot firmly and lift it",
1984
+ "source": {
1985
+ "instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/instruction.json",
1986
+ "transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/transcript.json"
1987
+ }
1988
  }
1989
  }
1990
  }