Update README.md
Browse files
README.md
CHANGED
|
@@ -72,18 +72,16 @@ vla = AutoModelForVision2Seq.from_pretrained(
|
|
| 72 |
low_cpu_mem_usage=True,
|
| 73 |
trust_remote_code=True
|
| 74 |
).to("cuda:0")
|
|
|
|
| 75 |
|
| 76 |
-
# Grab image input & format prompt of size 224x224
|
| 77 |
image: Image.Image = get_from_camera(...)
|
| 78 |
prompt = "In: What action should the robot take to achieve the instruction\nINSTRUCTION: \n{<Instruction here>}\n\nOut: "
|
| 79 |
|
| 80 |
# Predict Action (action is a 7 dimensional vector to control the robot)
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
max_new_tokens=512, do_sample=False
|
| 84 |
-
)
|
| 85 |
|
| 86 |
-
print("
|
| 87 |
# Execute...
|
| 88 |
robot.act(action, ...)
|
| 89 |
```
|
|
|
|
| 72 |
low_cpu_mem_usage=True,
|
| 73 |
trust_remote_code=True
|
| 74 |
).to("cuda:0")
|
| 75 |
+
processor = AutoProcessor.from_pretrained("declare-lab/Emma-X", trust_remote_code=True)
|
| 76 |
|
|
|
|
| 77 |
image: Image.Image = get_from_camera(...)
|
| 78 |
prompt = "In: What action should the robot take to achieve the instruction\nINSTRUCTION: \n{<Instruction here>}\n\nOut: "
|
| 79 |
|
| 80 |
# Predict Action (action is a 7 dimensional vector to control the robot)
|
| 81 |
+
inputs = processor(prompt, image).to("cuda:0", dtype=torch.bfloat16)
|
| 82 |
+
action, _ = vla.generate_actions(inputs, do_sample=False, max_new_tokens=512)
|
|
|
|
|
|
|
| 83 |
|
| 84 |
+
print("action", action)
|
| 85 |
# Execute...
|
| 86 |
robot.act(action, ...)
|
| 87 |
```
|