Update native Kinex Hard results: 9 completed valid episodes
Browse files- data-source.json +2 -2
- results/cost-data.json +232 -0
- results/data.json +0 -0
- results/measured-instructions.json +44 -0
data-source.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
{
|
| 2 |
"schema": "programvla/dataset-source/1",
|
| 3 |
"dataset": "RLE-Bench/programmable-vla",
|
| 4 |
-
"revision": "
|
| 5 |
-
"evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/
|
| 6 |
"files": {
|
| 7 |
"data.json": {
|
| 8 |
"url": "./results/data.json"
|
|
|
|
| 1 |
{
|
| 2 |
"schema": "programvla/dataset-source/1",
|
| 3 |
"dataset": "RLE-Bench/programmable-vla",
|
| 4 |
+
"revision": "7c93400e87c06e4f0bcd422c21a965e74893f361",
|
| 5 |
+
"evidence_root": "https://huggingface.co/datasets/RLE-Bench/programmable-vla/resolve/7c93400e87c06e4f0bcd422c21a965e74893f361/",
|
| 6 |
"files": {
|
| 7 |
"data.json": {
|
| 8 |
"url": "./results/data.json"
|
results/cost-data.json
CHANGED
|
@@ -7925,6 +7925,238 @@
|
|
| 7925 |
"long_context_responses": 0,
|
| 7926 |
"cache_write_assumed": false,
|
| 7927 |
"evidence_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/session.jsonl"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7928 |
}
|
| 7929 |
},
|
| 7930 |
"sources": [],
|
|
|
|
| 7925 |
"long_context_responses": 0,
|
| 7926 |
"cache_write_assumed": false,
|
| 7927 |
"evidence_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/session.jsonl"
|
| 7928 |
+
},
|
| 7929 |
+
"native-kinex-01-20261113": {
|
| 7930 |
+
"signature": [
|
| 7931 |
+
"native-kinex-01-20261113",
|
| 7932 |
+
"experiments/kinex-hard-native-20261113/native-kinex-01-20261113/usage.json",
|
| 7933 |
+
2555920,
|
| 7934 |
+
11275,
|
| 7935 |
+
2502272,
|
| 7936 |
+
0,
|
| 7937 |
+
null
|
| 7938 |
+
],
|
| 7939 |
+
"lower": 3.6025020000000003,
|
| 7940 |
+
"upper": 3.6025020000000003,
|
| 7941 |
+
"actual_billed_usd": null,
|
| 7942 |
+
"basis": "native_pi_session_api_equivalent",
|
| 7943 |
+
"buckets": [
|
| 7944 |
+
{
|
| 7945 |
+
"category": "fresh_input",
|
| 7946 |
+
"tier": "Recorded session",
|
| 7947 |
+
"tokens": 53648,
|
| 7948 |
+
"rate": 10.0,
|
| 7949 |
+
"rate_upper": 10.0,
|
| 7950 |
+
"lower": 0.5364800000000001,
|
| 7951 |
+
"upper": 0.5364800000000001
|
| 7952 |
+
},
|
| 7953 |
+
{
|
| 7954 |
+
"category": "cache_read",
|
| 7955 |
+
"tier": "Recorded session",
|
| 7956 |
+
"tokens": 2502272,
|
| 7957 |
+
"rate": 1.0,
|
| 7958 |
+
"rate_upper": 1.0,
|
| 7959 |
+
"lower": 2.502272,
|
| 7960 |
+
"upper": 2.502272
|
| 7961 |
+
},
|
| 7962 |
+
{
|
| 7963 |
+
"category": "cache_write",
|
| 7964 |
+
"tier": "Recorded session",
|
| 7965 |
+
"tokens": 0,
|
| 7966 |
+
"rate": null,
|
| 7967 |
+
"rate_upper": null,
|
| 7968 |
+
"lower": 0,
|
| 7969 |
+
"upper": 0
|
| 7970 |
+
},
|
| 7971 |
+
{
|
| 7972 |
+
"category": "output",
|
| 7973 |
+
"tier": "Recorded session",
|
| 7974 |
+
"tokens": 11275,
|
| 7975 |
+
"rate": 50.00000000000001,
|
| 7976 |
+
"rate_upper": 50.00000000000001,
|
| 7977 |
+
"lower": 0.5637500000000001,
|
| 7978 |
+
"upper": 0.5637500000000001
|
| 7979 |
+
}
|
| 7980 |
+
],
|
| 7981 |
+
"cache_lower": 2502272,
|
| 7982 |
+
"cache_upper": 2502272,
|
| 7983 |
+
"long_context_responses": 0,
|
| 7984 |
+
"cache_write_assumed": false,
|
| 7985 |
+
"evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/session.jsonl"
|
| 7986 |
+
},
|
| 7987 |
+
"native-kinex-03-20261113": {
|
| 7988 |
+
"signature": [
|
| 7989 |
+
"native-kinex-03-20261113",
|
| 7990 |
+
"experiments/kinex-hard-native-20261113/native-kinex-03-20261113/usage.json",
|
| 7991 |
+
2014191,
|
| 7992 |
+
9624,
|
| 7993 |
+
1972480,
|
| 7994 |
+
0,
|
| 7995 |
+
null
|
| 7996 |
+
],
|
| 7997 |
+
"lower": 2.87079,
|
| 7998 |
+
"upper": 2.87079,
|
| 7999 |
+
"actual_billed_usd": null,
|
| 8000 |
+
"basis": "native_pi_session_api_equivalent",
|
| 8001 |
+
"buckets": [
|
| 8002 |
+
{
|
| 8003 |
+
"category": "fresh_input",
|
| 8004 |
+
"tier": "Recorded session",
|
| 8005 |
+
"tokens": 41711,
|
| 8006 |
+
"rate": 10.0,
|
| 8007 |
+
"rate_upper": 10.0,
|
| 8008 |
+
"lower": 0.41711000000000004,
|
| 8009 |
+
"upper": 0.41711000000000004
|
| 8010 |
+
},
|
| 8011 |
+
{
|
| 8012 |
+
"category": "cache_read",
|
| 8013 |
+
"tier": "Recorded session",
|
| 8014 |
+
"tokens": 1972480,
|
| 8015 |
+
"rate": 1.0,
|
| 8016 |
+
"rate_upper": 1.0,
|
| 8017 |
+
"lower": 1.97248,
|
| 8018 |
+
"upper": 1.97248
|
| 8019 |
+
},
|
| 8020 |
+
{
|
| 8021 |
+
"category": "cache_write",
|
| 8022 |
+
"tier": "Recorded session",
|
| 8023 |
+
"tokens": 0,
|
| 8024 |
+
"rate": null,
|
| 8025 |
+
"rate_upper": null,
|
| 8026 |
+
"lower": 0,
|
| 8027 |
+
"upper": 0
|
| 8028 |
+
},
|
| 8029 |
+
{
|
| 8030 |
+
"category": "output",
|
| 8031 |
+
"tier": "Recorded session",
|
| 8032 |
+
"tokens": 9624,
|
| 8033 |
+
"rate": 50.0,
|
| 8034 |
+
"rate_upper": 50.0,
|
| 8035 |
+
"lower": 0.4812,
|
| 8036 |
+
"upper": 0.4812
|
| 8037 |
+
}
|
| 8038 |
+
],
|
| 8039 |
+
"cache_lower": 1972480,
|
| 8040 |
+
"cache_upper": 1972480,
|
| 8041 |
+
"long_context_responses": 0,
|
| 8042 |
+
"cache_write_assumed": false,
|
| 8043 |
+
"evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/session.jsonl"
|
| 8044 |
+
},
|
| 8045 |
+
"native-kinex-05-20261113": {
|
| 8046 |
+
"signature": [
|
| 8047 |
+
"native-kinex-05-20261113",
|
| 8048 |
+
"experiments/kinex-hard-native-20261113/native-kinex-05-20261113/usage.json",
|
| 8049 |
+
2051199,
|
| 8050 |
+
9278,
|
| 8051 |
+
2002432,
|
| 8052 |
+
0,
|
| 8053 |
+
null
|
| 8054 |
+
],
|
| 8055 |
+
"lower": 2.954002,
|
| 8056 |
+
"upper": 2.954002,
|
| 8057 |
+
"actual_billed_usd": null,
|
| 8058 |
+
"basis": "native_pi_session_api_equivalent",
|
| 8059 |
+
"buckets": [
|
| 8060 |
+
{
|
| 8061 |
+
"category": "fresh_input",
|
| 8062 |
+
"tier": "Recorded session",
|
| 8063 |
+
"tokens": 48767,
|
| 8064 |
+
"rate": 10.0,
|
| 8065 |
+
"rate_upper": 10.0,
|
| 8066 |
+
"lower": 0.48767000000000005,
|
| 8067 |
+
"upper": 0.48767000000000005
|
| 8068 |
+
},
|
| 8069 |
+
{
|
| 8070 |
+
"category": "cache_read",
|
| 8071 |
+
"tier": "Recorded session",
|
| 8072 |
+
"tokens": 2002432,
|
| 8073 |
+
"rate": 1.0,
|
| 8074 |
+
"rate_upper": 1.0,
|
| 8075 |
+
"lower": 2.0024319999999998,
|
| 8076 |
+
"upper": 2.0024319999999998
|
| 8077 |
+
},
|
| 8078 |
+
{
|
| 8079 |
+
"category": "cache_write",
|
| 8080 |
+
"tier": "Recorded session",
|
| 8081 |
+
"tokens": 0,
|
| 8082 |
+
"rate": null,
|
| 8083 |
+
"rate_upper": null,
|
| 8084 |
+
"lower": 0,
|
| 8085 |
+
"upper": 0
|
| 8086 |
+
},
|
| 8087 |
+
{
|
| 8088 |
+
"category": "output",
|
| 8089 |
+
"tier": "Recorded session",
|
| 8090 |
+
"tokens": 9278,
|
| 8091 |
+
"rate": 50.0,
|
| 8092 |
+
"rate_upper": 50.0,
|
| 8093 |
+
"lower": 0.46390000000000003,
|
| 8094 |
+
"upper": 0.46390000000000003
|
| 8095 |
+
}
|
| 8096 |
+
],
|
| 8097 |
+
"cache_lower": 2002432,
|
| 8098 |
+
"cache_upper": 2002432,
|
| 8099 |
+
"long_context_responses": 0,
|
| 8100 |
+
"cache_write_assumed": false,
|
| 8101 |
+
"evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/session.jsonl"
|
| 8102 |
+
},
|
| 8103 |
+
"native-kinex-vlact50k-10-20261113": {
|
| 8104 |
+
"signature": [
|
| 8105 |
+
"native-kinex-vlact50k-10-20261113",
|
| 8106 |
+
"experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/usage.json",
|
| 8107 |
+
946198,
|
| 8108 |
+
4960,
|
| 8109 |
+
911744,
|
| 8110 |
+
0,
|
| 8111 |
+
null
|
| 8112 |
+
],
|
| 8113 |
+
"lower": 1.504284,
|
| 8114 |
+
"upper": 1.504284,
|
| 8115 |
+
"actual_billed_usd": null,
|
| 8116 |
+
"basis": "native_pi_session_api_equivalent",
|
| 8117 |
+
"buckets": [
|
| 8118 |
+
{
|
| 8119 |
+
"category": "fresh_input",
|
| 8120 |
+
"tier": "Recorded session",
|
| 8121 |
+
"tokens": 34454,
|
| 8122 |
+
"rate": 10.0,
|
| 8123 |
+
"rate_upper": 10.0,
|
| 8124 |
+
"lower": 0.34454,
|
| 8125 |
+
"upper": 0.34454
|
| 8126 |
+
},
|
| 8127 |
+
{
|
| 8128 |
+
"category": "cache_read",
|
| 8129 |
+
"tier": "Recorded session",
|
| 8130 |
+
"tokens": 911744,
|
| 8131 |
+
"rate": 1.0,
|
| 8132 |
+
"rate_upper": 1.0,
|
| 8133 |
+
"lower": 0.911744,
|
| 8134 |
+
"upper": 0.911744
|
| 8135 |
+
},
|
| 8136 |
+
{
|
| 8137 |
+
"category": "cache_write",
|
| 8138 |
+
"tier": "Recorded session",
|
| 8139 |
+
"tokens": 0,
|
| 8140 |
+
"rate": null,
|
| 8141 |
+
"rate_upper": null,
|
| 8142 |
+
"lower": 0,
|
| 8143 |
+
"upper": 0
|
| 8144 |
+
},
|
| 8145 |
+
{
|
| 8146 |
+
"category": "output",
|
| 8147 |
+
"tier": "Recorded session",
|
| 8148 |
+
"tokens": 4960,
|
| 8149 |
+
"rate": 50.0,
|
| 8150 |
+
"rate_upper": 50.0,
|
| 8151 |
+
"lower": 0.24800000000000003,
|
| 8152 |
+
"upper": 0.24800000000000003
|
| 8153 |
+
}
|
| 8154 |
+
],
|
| 8155 |
+
"cache_lower": 911744,
|
| 8156 |
+
"cache_upper": 911744,
|
| 8157 |
+
"long_context_responses": 0,
|
| 8158 |
+
"cache_write_assumed": false,
|
| 8159 |
+
"evidence_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/session.jsonl"
|
| 8160 |
}
|
| 8161 |
},
|
| 8162 |
"sources": [],
|
results/data.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/measured-instructions.json
CHANGED
|
@@ -1941,6 +1941,50 @@
|
|
| 1941 |
"instructions_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/instruction.json",
|
| 1942 |
"transcript_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/transcript.json"
|
| 1943 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1944 |
}
|
| 1945 |
}
|
| 1946 |
}
|
|
|
|
| 1941 |
"instructions_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/instruction.json",
|
| 1942 |
"transcript_url": "experiments/kinex-pose-hard-native-20261113/native-full-kinex-tcp-posestate-07-20261113/transcript.json"
|
| 1943 |
}
|
| 1944 |
+
},
|
| 1945 |
+
"native-kinex-01-20261113": {
|
| 1946 |
+
"task_key": "task03/01",
|
| 1947 |
+
"initial_goal": "Grab the red block with the left arm, transfer it to the right arm, and set it on the blue pad.",
|
| 1948 |
+
"text": "Grab the red block with the left arm, transfer it to the right arm, and set it on the blue pad.",
|
| 1949 |
+
"kind": "task_instruction",
|
| 1950 |
+
"observed_native": "Grab the red block with the left arm, transfer it to the right arm, and set it on the blue pad.",
|
| 1951 |
+
"source": {
|
| 1952 |
+
"instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/instruction.json",
|
| 1953 |
+
"transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-01-20261113/transcript.json"
|
| 1954 |
+
}
|
| 1955 |
+
},
|
| 1956 |
+
"native-kinex-03-20261113": {
|
| 1957 |
+
"task_key": "task03/03",
|
| 1958 |
+
"initial_goal": "Place the yellow bottle with elongated cylinder body, the bottle with blue highlights, and the bottle with red screw cap in the dustbin the simple black garbage bin.",
|
| 1959 |
+
"text": "Place the yellow bottle with elongated cylinder body, the bottle with blue highlights, and the bottle with red screw cap in the dustbin the simple black garbage bin.",
|
| 1960 |
+
"kind": "task_instruction",
|
| 1961 |
+
"observed_native": "Place the yellow bottle with elongated cylinder body, the bottle with blue highlights, and the bottle with red screw cap in the dustbin the simple black garbage bin.",
|
| 1962 |
+
"source": {
|
| 1963 |
+
"instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/instruction.json",
|
| 1964 |
+
"transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-03-20261113/transcript.json"
|
| 1965 |
+
}
|
| 1966 |
+
},
|
| 1967 |
+
"native-kinex-05-20261113": {
|
| 1968 |
+
"task_key": "task03/05",
|
| 1969 |
+
"initial_goal": "Grab the smooth bowl with glossy finish, stack them one by one.",
|
| 1970 |
+
"text": "Grab the smooth bowl with glossy finish, stack them one by one.",
|
| 1971 |
+
"kind": "task_instruction",
|
| 1972 |
+
"observed_native": "Grab the smooth bowl with glossy finish, stack them one by one.",
|
| 1973 |
+
"source": {
|
| 1974 |
+
"instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/instruction.json",
|
| 1975 |
+
"transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-05-20261113/transcript.json"
|
| 1976 |
+
}
|
| 1977 |
+
},
|
| 1978 |
+
"native-kinex-vlact50k-10-20261113": {
|
| 1979 |
+
"task_key": "task03/10",
|
| 1980 |
+
"initial_goal": "Hold the dark gray cooking kitchenpot firmly and lift it",
|
| 1981 |
+
"text": "Hold the dark gray cooking kitchenpot firmly and lift it",
|
| 1982 |
+
"kind": "task_instruction",
|
| 1983 |
+
"observed_native": "Hold the dark gray cooking kitchenpot firmly and lift it",
|
| 1984 |
+
"source": {
|
| 1985 |
+
"instructions_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/instruction.json",
|
| 1986 |
+
"transcript_url": "experiments/kinex-hard-native-20261113/native-kinex-vlact50k-10-20261113/transcript.json"
|
| 1987 |
+
}
|
| 1988 |
}
|
| 1989 |
}
|
| 1990 |
}
|