Upload folder using huggingface_hub

.gitattributes

@@ -208,3 +208,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 2025.03.27/17.16.51_train_llm_lowdim_box-close-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-box-close-v2/checkpoint-600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 2025.03.27/17.16.51_train_llm_lowdim_box-close-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-box-close-v2/checkpoint-800/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 2025.03.27/17.16.51_train_llm_lowdim_box-close-v2/wandb/run-20250327_171652-6gwnpbyj/run-6gwnpbyj.wandb filter=lfs diff=lfs merge=lfs -text
+2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/run-0qs6crol.wandb filter=lfs diff=lfs merge=lfs -text

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/.hydra/config.yaml

@@ -0,0 +1,139 @@
+name: train_mlp_projector
+_target_: llmbc.workspace.train_mlp_projector_workspace.TrainMlpProjectorWorkspace
+obs_dim: ${task.obs_dim}
+action_dim: ${task.action_dim}
+task_name: ${task.name}
+exp_name: default
+model_name: ${llm.name}
+horizon: 1
+n_obs_steps: 1
+n_action_steps: 1
+n_latency_steps: 0
+past_action_visible: false
+llm_translator:
+  _target_: llmbc.translator.llm_translator.LLMTranslator
+  cfg: ${llm}
+  obs_dim: ${task.obs_dim}
+  action_dim: ${task.action_dim}
+  horizon: ${horizon}
+  n_obs_steps: ${n_obs_steps}
+  n_action_steps: ${n_action_steps}
+dataloader:
+  batch_size: 8
+  num_workers: 0
+  shuffle: true
+  pin_memory: false
+  persistent_workers: false
+val_dataloader:
+  batch_size: 8
+  num_workers: 0
+  shuffle: true
+  pin_memory: false
+  persistent_workers: false
+optimizer:
+  _target_: torch.optim.Adam
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda
+  seed: 42
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 10
+  num_epochs: 10
+  gradient_accumulate_every: 64
+  grad_norm_clip: 1
+  enable_normalizer: true
+  checkpoint_every: 1
+  val_every: 1
+  sample_every: 1
+  sample_max_batch: 128
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: llm_module_training
+  resume: true
+  mode: online
+  name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: null
+  group: null
+checkpoint:
+  topk:
+    monitor_key: val_loss
+    mode: min
+    k: 5
+    format_str: epoch={epoch:04d}-val_loss={val_loss:.3f}.ckpt
+  save_last_ckpt: true
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: blockpush_lowdim_seed
+  obs_dim: 16
+  action_dim: 2
+  keypoint_dim: 2
+  obs_eef_target: true
+  env_runner:
+    _target_: llmbc.env_runner.blockpushing_lowdim_runner.BlockPushingLowdimRunner
+    env_name: llf-blockpushing-BlockPushMultimodal-v0
+    instruction_type: b
+    feedback_type:
+    - hp
+    - hn
+    - fp
+    visual: false
+    max_steps: 350
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+  dataset:
+    _target_: llmbc.dataset.blockpush_lowdim_dataset.BlockPushLowdimDataset
+    data_path: datasets/BlockPushMultimodal-v0-general.pt
+    data_path2: datasets/BlockPushMultimodal-v0.pt
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    obs_eef_target: ${task.obs_eef_target}
+    use_manual_normalizer: false
+    seed: 42
+    val_ratio: 0.02
+    dummy_normalizer: false
+  instructor:
+    _target_: llmbc.translator.instructor.blockpush_instructor.BlockPushMultimodal_v0_instructor.BlockPushMultimodalV0Instructor
+llm:
+  name: meta-llama/Llama-3.2-1B-Instruct
+  model_name: Llama-3.2-1B-Instruct
+  use_quantization: false
+  load_from_checkpoint: false
+  adaptor_path: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100
+  use_orig_model: false
+  use_joint_mlp_projector: true
+  load_from_mlp_projector_checkpoint: false
+  mlp_projector_checkpoint_path: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin
+  max_length: 100
+  config_target: llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig
+  causal_lm_target: llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM
+  lora_config:
+    r: 16
+    lora_alpha: 32
+    lora_dropout: 0.05
+    bias: none
+    task_type: CAUSAL_LM
+  prompter:
+    _target_: llmbc.translator.prompter.llama_prompter.LlamaPrompter
+    use_joint_mlp_projector: true
+  hydra:
+    job:
+      override_dirname: ${model_name}
+    run:
+      dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${model_name}

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/.hydra/hydra.yaml

@@ -0,0 +1,158 @@
+hydra:
+  run:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - llm.load_from_checkpoint=False
+    - llm.load_from_mlp_projector_checkpoint=False
+  job:
+    name: train_mlp_projector_workspace
+    chdir: null
+    override_dirname: llm.load_from_checkpoint=False,llm.load_from_mlp_projector_checkpoint=False
+    id: ???
+    num: ???
+    config_name: train_mlp_projector_workspace
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /home/chyang/workspace/LLM-BC
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/chyang/workspace/LLM-BC/llmbc/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed
+    choices:
+      llm: llama-3.2-1b-instruct
+      task: BlockPushMultimodal-v0
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- llm.load_from_checkpoint=False
+- llm.load_from_mlp_projector_checkpoint=False

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/checkpoints/epoch=0005-val_loss=0.655.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6619549f593574d9474b28d738e292df5c1851bd4dea2f5401beb5bcd6515199
+size 50851650

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/checkpoints/epoch=0006-val_loss=0.654.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cad581b55b513516dcb43ef59cdcb9fff0e2b1e73b0098a4a544f709aac76049
+size 50851650

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/checkpoints/epoch=0007-val_loss=0.631.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:253ae2c52afe78b079da38480345c9ca8901c6163389139c34502edb68d2c19b
+size 50851650

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/checkpoints/epoch=0008-val_loss=0.626.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d368a990898964263de0342b03ca0c42d4f2ab615a1ed8825d7a482973966aa6
+size 50851650

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/checkpoints/epoch=0009-val_loss=0.638.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03cc9e623600b00cfb258baeebb7b5d34771b8ee27a57bd0cac25b554a871aa4
+size 50851650

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/checkpoints/latest.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03cc9e623600b00cfb258baeebb7b5d34771b8ee27a57bd0cac25b554a871aa4
+size 50851650

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24982c0392c43659fa27ce95bdddb0388b78ef863192741ff0aa62e684c818af
+size 40444501

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/debug-internal.log

@@ -0,0 +1,25 @@
+{"time":"2025-03-27T17:04:44.263776842+08:00","level":"INFO","msg":"using version","core version":"0.18.6"}
+{"time":"2025-03-27T17:04:44.263801603+08:00","level":"INFO","msg":"created symlink","path":"/home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug-core.log"}
+{"time":"2025-03-27T17:04:44.368088929+08:00","level":"INFO","msg":"created new stream","id":"0qs6crol"}
+{"time":"2025-03-27T17:04:44.368115827+08:00","level":"INFO","msg":"stream: started","id":"0qs6crol"}
+{"time":"2025-03-27T17:04:44.368164562+08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-27T17:04:44.368295512+08:00","level":"INFO","msg":"sender: started","stream_id":"0qs6crol"}
+{"time":"2025-03-27T17:04:44.368175374+08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-27T17:04:44.980715375+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-27T20:36:45.696778412+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream"}
+{"time":"2025-03-27T22:02:15.257097354+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-03-27T22:02:47.376446372+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-03-27T22:03:44.135456734+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
+{"time":"2025-03-27T23:51:45.33277774+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-03-27T23:52:17.477495599+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-03-27T23:52:51.994006128+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-03-27T23:53:28.455445934+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
+{"time":"2025-03-28T00:41:16.598881745+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream"}
+{"time":"2025-03-28T10:26:58.369916065+08:00","level":"INFO","msg":"stream: closing","id":"0qs6crol"}
+{"time":"2025-03-28T10:26:58.369969281+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-03-28T10:26:58.370920003+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-03-28T10:27:01.801436411+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-03-28T10:27:02.03763749+08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-28T10:27:02.0377383+08:00","level":"INFO","msg":"sender: closed","stream_id":"0qs6crol"}
+{"time":"2025-03-28T10:27:02.037711532+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-28T10:27:02.037855219+08:00","level":"INFO","msg":"stream: closed","id":"0qs6crol"}

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/debug.log

@@ -0,0 +1,27 @@
+2025-03-27 17:04:44,257 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Current SDK version is 0.18.6
+2025-03-27 17:04:44,257 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Configure stats pid to 1469429
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Loading settings from /home/chyang/.config/wandb/settings
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Loading settings from /home/chyang/workspace/LLM-BC/wandb/settings
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'llmbc/workspace/train_mlp_projector_workspace.py', 'program_abspath': '/home/chyang/workspace/LLM-BC/llmbc/workspace/train_mlp_projector_workspace.py', 'program': '/home/chyang/workspace/LLM-BC/./llmbc/workspace/train_mlp_projector_workspace.py'}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Applying login settings: {}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:_log_setup():533] Logging user logs to /home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug.log
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:_log_setup():534] Logging internal logs to /home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug-internal.log
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():619] calling init triggers
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'name': 'train_mlp_projector', '_target_': 'llmbc.workspace.train_mlp_projector_workspace.TrainMlpProjectorWorkspace', 'obs_dim': 16, 'action_dim': 2, 'task_name': 'blockpush_lowdim_seed', 'exp_name': 'default', 'model_name': 'meta-llama/Llama-3.2-1B-Instruct', 'horizon': 1, 'n_obs_steps': 1, 'n_action_steps': 1, 'n_latency_steps': 0, 'past_action_visible': False, 'llm_translator': {'_target_': 'llmbc.translator.llm_translator.LLMTranslator', 'cfg': {'name': 'meta-llama/Llama-3.2-1B-Instruct', 'model_name': 'Llama-3.2-1B-Instruct', 'use_quantization': False, 'load_from_checkpoint': False, 'adaptor_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100', 'use_orig_model': False, 'use_joint_mlp_projector': True, 'load_from_mlp_projector_checkpoint': False, 'mlp_projector_checkpoint_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin', 'max_length': 100, 'config_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig', 'causal_lm_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM', 'lora_config': {'r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'bias': 'none', 'task_type': 'CAUSAL_LM'}, 'prompter': {'_target_': 'llmbc.translator.prompter.llama_prompter.LlamaPrompter', 'use_joint_mlp_projector': True}, 'hydra': {'job': {'override_dirname': 'meta-llama/Llama-3.2-1B-Instruct'}, 'run': {'dir': 'data/outputs/2025.03.27/17.04.38_meta-llama/Llama-3.2-1B-Instruct'}}}, 'obs_dim': 16, 'action_dim': 2, 'horizon': 1, 'n_obs_steps': 1, 'n_action_steps': 1}, 'dataloader': {'batch_size': 8, 'num_workers': 0, 'shuffle': True, 'pin_memory': False, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 8, 'num_workers': 0, 'shuffle': True, 'pin_memory': False, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.Adam', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda', 'seed': 42, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 10, 'num_epochs': 10, 'gradient_accumulate_every': 64, 'grad_norm_clip': 1, 'enable_normalizer': True, 'checkpoint_every': 1, 'val_every': 1, 'sample_every': 1, 'sample_max_batch': 128, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'project': 'llm_module_training', 'resume': True, 'mode': 'online', 'name': '2025.03.27-17.04.43_train_mlp_projector_blockpush_lowdim_seed', 'tags': ['train_mlp_projector', 'blockpush_lowdim_seed', 'default'], 'id': None, 'group': None}, 'checkpoint': {'topk': {'monitor_key': 'val_loss', 'mode': 'min', 'k': 5, 'format_str': 'epoch={epoch:04d}-val_loss={val_loss:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed', 'wandb_name_base': '2025.03.27-17.04.43_train_mlp_projector_blockpush_lowdim_seed'}, 'task': {'name': 'blockpush_lowdim_seed', 'obs_dim': 16, 'action_dim': 2, 'keypoint_dim': 2, 'obs_eef_target': True, 'env_runner': {'_target_': 'llmbc.env_runner.blockpushing_lowdim_runner.BlockPushingLowdimRunner', 'env_name': 'llf-blockpushing-BlockPushMultimodal-v0', 'instruction_type': 'b', 'feedback_type': ['hp', 'hn', 'fp'], 'visual': False, 'max_steps': 350, 'n_obs_steps': 1, 'n_action_steps': 1}, 'dataset': {'_target_': 'llmbc.dataset.blockpush_lowdim_dataset.BlockPushLowdimDataset', 'data_path': 'datasets/BlockPushMultimodal-v0-general.pt', 'data_path2': 'datasets/BlockPushMultimodal-v0.pt', 'horizon': 1, 'pad_before': 0, 'pad_after': 0, 'obs_eef_target': True, 'use_manual_normalizer': False, 'seed': 42, 'val_ratio': 0.02, 'dummy_normalizer': False}, 'instructor': {'_target_': 'llmbc.translator.instructor.blockpush_instructor.BlockPushMultimodal_v0_instructor.BlockPushMultimodalV0Instructor'}}, 'llm': {'name': 'meta-llama/Llama-3.2-1B-Instruct', 'model_name': 'Llama-3.2-1B-Instruct', 'use_quantization': False, 'load_from_checkpoint': False, 'adaptor_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100', 'use_orig_model': False, 'use_joint_mlp_projector': True, 'load_from_mlp_projector_checkpoint': False, 'mlp_projector_checkpoint_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin', 'max_length': 100, 'config_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig', 'causal_lm_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM', 'lora_config': {'r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'bias': 'none', 'task_type': 'CAUSAL_LM'}, 'prompter': {'_target_': 'llmbc.translator.prompter.llama_prompter.LlamaPrompter', 'use_joint_mlp_projector': True}, 'hydra': {'job': {'override_dirname': 'meta-llama/Llama-3.2-1B-Instruct'}, 'run': {'dir': 'data/outputs/2025.03.27/17.04.38_meta-llama/Llama-3.2-1B-Instruct'}}}}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():669] starting backend
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():673] sending inform_init request
+2025-03-27 17:04:44,259 INFO    MainThread:1469429 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-27 17:04:44,259 INFO    MainThread:1469429 [wandb_init.py:init():686] backend started and connected
+2025-03-27 17:04:44,264 INFO    MainThread:1469429 [wandb_init.py:init():781] updated telemetry
+2025-03-27 17:04:44,322 INFO    MainThread:1469429 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2025-03-27 17:04:44,978 INFO    MainThread:1469429 [wandb_init.py:init():867] starting run threads in backend
+2025-03-27 17:04:45,079 INFO    MainThread:1469429 [wandb_run.py:_console_start():2451] atexit reg
+2025-03-27 17:04:45,079 INFO    MainThread:1469429 [wandb_run.py:_redirect():2299] redirect: wrap_raw
+2025-03-27 17:04:45,081 INFO    MainThread:1469429 [wandb_run.py:_redirect():2364] Wrapping output streams.
+2025-03-27 17:04:45,081 INFO    MainThread:1469429 [wandb_run.py:_redirect():2389] Redirects installed.
+2025-03-27 17:04:45,082 INFO    MainThread:1469429 [wandb_init.py:init():911] run started, returning control to user process
+2025-03-27 17:04:45,083 INFO    MainThread:1469429 [wandb_run.py:_config_callback():1389] config_cb None None {'output_dir': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed'}
+2025-03-28 10:26:58,370 WARNING MsgRouterThr:1469429 [router.py:message_loop():75] message_loop has been closed

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/files/config.yaml

@@ -0,0 +1,241 @@
+_target_:
+    value: llmbc.workspace.train_mlp_projector_workspace.TrainMlpProjectorWorkspace
+_wandb:
+    value:
+        cli_version: 0.18.6
+        m: []
+        python_version: 3.9.21
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 12
+                - 41
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+                - 71
+                - 83
+                - 98
+            "2":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 12
+                - 41
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+                - 71
+                - 83
+                - 98
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.9.21
+            "5": 0.18.6
+            "6": 4.47.1
+            "8":
+                - 5
+            "12": 0.18.6
+            "13": linux-x86_64
+action_dim:
+    value: 2
+checkpoint:
+    value:
+        save_last_ckpt: true
+        save_last_snapshot: false
+        topk:
+            format_str: epoch={epoch:04d}-val_loss={val_loss:.3f}.ckpt
+            k: 5
+            mode: min
+            monitor_key: val_loss
+dataloader:
+    value:
+        batch_size: 8
+        num_workers: 0
+        persistent_workers: false
+        pin_memory: false
+        shuffle: true
+exp_name:
+    value: default
+horizon:
+    value: 1
+llm:
+    value:
+        adaptor_path: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100
+        causal_lm_target: llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM
+        config_target: llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig
+        hydra:
+            job:
+                override_dirname: meta-llama/Llama-3.2-1B-Instruct
+            run:
+                dir: data/outputs/2025.03.27/17.04.38_meta-llama/Llama-3.2-1B-Instruct
+        load_from_checkpoint: false
+        load_from_mlp_projector_checkpoint: false
+        lora_config:
+            bias: none
+            lora_alpha: 32
+            lora_dropout: 0.05
+            r: 16
+            task_type: CAUSAL_LM
+        max_length: 100
+        mlp_projector_checkpoint_path: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin
+        model_name: Llama-3.2-1B-Instruct
+        name: meta-llama/Llama-3.2-1B-Instruct
+        prompter:
+            _target_: llmbc.translator.prompter.llama_prompter.LlamaPrompter
+            use_joint_mlp_projector: true
+        use_joint_mlp_projector: true
+        use_orig_model: false
+        use_quantization: false
+llm_translator:
+    value:
+        _target_: llmbc.translator.llm_translator.LLMTranslator
+        action_dim: 2
+        cfg:
+            adaptor_path: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100
+            causal_lm_target: llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM
+            config_target: llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig
+            hydra:
+                job:
+                    override_dirname: meta-llama/Llama-3.2-1B-Instruct
+                run:
+                    dir: data/outputs/2025.03.27/17.04.38_meta-llama/Llama-3.2-1B-Instruct
+            load_from_checkpoint: false
+            load_from_mlp_projector_checkpoint: false
+            lora_config:
+                bias: none
+                lora_alpha: 32
+                lora_dropout: 0.05
+                r: 16
+                task_type: CAUSAL_LM
+            max_length: 100
+            mlp_projector_checkpoint_path: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin
+            model_name: Llama-3.2-1B-Instruct
+            name: meta-llama/Llama-3.2-1B-Instruct
+            prompter:
+                _target_: llmbc.translator.prompter.llama_prompter.LlamaPrompter
+                use_joint_mlp_projector: true
+            use_joint_mlp_projector: true
+            use_orig_model: false
+            use_quantization: false
+        horizon: 1
+        n_action_steps: 1
+        n_obs_steps: 1
+        obs_dim: 16
+logging:
+    value:
+        group: null
+        id: null
+        mode: online
+        name: 2025.03.27-17.04.43_train_mlp_projector_blockpush_lowdim_seed
+        project: llm_module_training
+        resume: true
+        tags:
+            - train_mlp_projector
+            - blockpush_lowdim_seed
+            - default
+model_name:
+    value: meta-llama/Llama-3.2-1B-Instruct
+multi_run:
+    value:
+        run_dir: data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed
+        wandb_name_base: 2025.03.27-17.04.43_train_mlp_projector_blockpush_lowdim_seed
+n_action_steps:
+    value: 1
+n_latency_steps:
+    value: 0
+n_obs_steps:
+    value: 1
+name:
+    value: train_mlp_projector
+obs_dim:
+    value: 16
+optimizer:
+    value:
+        _target_: torch.optim.Adam
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        lr: 0.0001
+        weight_decay: 1e-06
+output_dir:
+    value: /home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed
+past_action_visible:
+    value: false
+task:
+    value:
+        action_dim: 2
+        dataset:
+            _target_: llmbc.dataset.blockpush_lowdim_dataset.BlockPushLowdimDataset
+            data_path: datasets/BlockPushMultimodal-v0-general.pt
+            data_path2: datasets/BlockPushMultimodal-v0.pt
+            dummy_normalizer: false
+            horizon: 1
+            obs_eef_target: true
+            pad_after: 0
+            pad_before: 0
+            seed: 42
+            use_manual_normalizer: false
+            val_ratio: 0.02
+        env_runner:
+            _target_: llmbc.env_runner.blockpushing_lowdim_runner.BlockPushingLowdimRunner
+            env_name: llf-blockpushing-BlockPushMultimodal-v0
+            feedback_type:
+                - hp
+                - hn
+                - fp
+            instruction_type: b
+            max_steps: 350
+            n_action_steps: 1
+            n_obs_steps: 1
+            visual: false
+        instructor:
+            _target_: llmbc.translator.instructor.blockpush_instructor.BlockPushMultimodal_v0_instructor.BlockPushMultimodalV0Instructor
+        keypoint_dim: 2
+        name: blockpush_lowdim_seed
+        obs_dim: 16
+        obs_eef_target: true
+task_name:
+    value: blockpush_lowdim_seed
+training:
+    value:
+        checkpoint_every: 1
+        debug: false
+        device: cuda
+        enable_normalizer: true
+        grad_norm_clip: 1
+        gradient_accumulate_every: 64
+        lr_scheduler: cosine
+        lr_warmup_steps: 10
+        max_train_steps: null
+        max_val_steps: null
+        num_epochs: 10
+        resume: true
+        sample_every: 1
+        sample_max_batch: 128
+        seed: 42
+        tqdm_interval_sec: 1
+        val_every: 1
+val_dataloader:
+    value:
+        batch_size: 8
+        num_workers: 0
+        persistent_workers: false
+        pin_memory: false
+        shuffle: true

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/files/output.log

@@ -0,0 +1 @@
+                                                                                                                                                                                      

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/files/wandb-metadata.json

@@ -0,0 +1,53 @@
+{
+  "os": "Linux-5.15.0-101-generic-x86_64-with-glibc2.31",
+  "python": "3.9.21",
+  "startedAt": "2025-03-27T09:04:44.259481Z",
+  "args": [
+    "llm.load_from_checkpoint=False",
+    "llm.load_from_mlp_projector_checkpoint=False"
+  ],
+  "program": "/home/chyang/workspace/LLM-BC/./llmbc/workspace/train_mlp_projector_workspace.py",
+  "codePath": "llmbc/workspace/train_mlp_projector_workspace.py",
+  "git": {
+    "remote": "https://github.com/CHYang25/LLM-BC.git",
+    "commit": "23c440e8691994ef89808e56780631cb70d740cd"
+  },
+  "email": "chris920325@gmail.com",
+  "root": "/home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed",
+  "host": "user",
+  "username": "chyang",
+  "executable": "/home/chyang/miniconda3/envs/llm-bc/bin/python3",
+  "codePathLocal": "llmbc/workspace/train_mlp_projector_workspace.py",
+  "cpu_count": 20,
+  "cpu_count_logical": 40,
+  "gpu": "NVIDIA GeForce RTX 4090",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1967317549056",
+      "used": "1723504910336"
+    }
+  },
+  "memory": {
+    "total": "134536400896"
+  },
+  "cpu": {
+    "count": 20,
+    "countLogical": 40
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    }
+  ],
+  "cudaVersion": "12.2"
+}

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/files/wandb-summary.json

@@ -0,0 +1 @@
+{"global_step":307539,"grad_norm":0.9457659125328064,"lr":1.0731551536302675e-11,"val_loss":0.6377049684524536,"train_loss":0.6319962019829878,"_wandb":{"runtime":62534},"_runtime":62534.110468589,"_timestamp":1.7431288182431498e+09,"_step":307539,"epoch":9}

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-03-27T17:04:43.68351449+08:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpcn3l42ew/port-1469429.txt","pid":1469429,"debug":false,"disable-analytics":false}
+{"time":"2025-03-27T17:04:43.683533923+08:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2025-03-27T17:04:43.684676388+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33817,"Zone":""}}
+{"time":"2025-03-27T17:04:43.6847039+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":1469429}
+{"time":"2025-03-27T17:04:43.878701255+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:41792"}
+{"time":"2025-03-27T17:04:44.263499439+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"0qs6crol","id":"127.0.0.1:41792"}
+{"time":"2025-03-27T17:04:44.36812052+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"0qs6crol","id":"127.0.0.1:41792"}
+{"time":"2025-03-28T10:26:58.369784498+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:41792"}
+{"time":"2025-03-28T10:26:58.36997353+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-03-28T10:26:58.36995395+08:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:41792"}
+{"time":"2025-03-28T10:26:58.370151439+08:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:41792"}
+{"time":"2025-03-28T10:27:02.037939133+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:41792"}
+{"time":"2025-03-28T10:27:02.037978805+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:41792"}
+{"time":"2025-03-28T10:27:02.038001492+08:00","level":"INFO","msg":"server is closed"}

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug-internal.log

@@ -0,0 +1,25 @@
+{"time":"2025-03-27T17:04:44.263776842+08:00","level":"INFO","msg":"using version","core version":"0.18.6"}
+{"time":"2025-03-27T17:04:44.263801603+08:00","level":"INFO","msg":"created symlink","path":"/home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug-core.log"}
+{"time":"2025-03-27T17:04:44.368088929+08:00","level":"INFO","msg":"created new stream","id":"0qs6crol"}
+{"time":"2025-03-27T17:04:44.368115827+08:00","level":"INFO","msg":"stream: started","id":"0qs6crol"}
+{"time":"2025-03-27T17:04:44.368164562+08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-27T17:04:44.368295512+08:00","level":"INFO","msg":"sender: started","stream_id":"0qs6crol"}
+{"time":"2025-03-27T17:04:44.368175374+08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-27T17:04:44.980715375+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-27T20:36:45.696778412+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream"}
+{"time":"2025-03-27T22:02:15.257097354+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-03-27T22:02:47.376446372+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-03-27T22:03:44.135456734+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
+{"time":"2025-03-27T23:51:45.33277774+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-03-27T23:52:17.477495599+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-03-27T23:52:51.994006128+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-03-27T23:53:28.455445934+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
+{"time":"2025-03-28T00:41:16.598881745+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/chyang25-national-taiwan-university/llm_module_training/0qs6crol/file_stream"}
+{"time":"2025-03-28T10:26:58.369916065+08:00","level":"INFO","msg":"stream: closing","id":"0qs6crol"}
+{"time":"2025-03-28T10:26:58.369969281+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-03-28T10:26:58.370920003+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-03-28T10:27:01.801436411+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-03-28T10:27:02.03763749+08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-28T10:27:02.0377383+08:00","level":"INFO","msg":"sender: closed","stream_id":"0qs6crol"}
+{"time":"2025-03-28T10:27:02.037711532+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"0qs6crol"}}
+{"time":"2025-03-28T10:27:02.037855219+08:00","level":"INFO","msg":"stream: closed","id":"0qs6crol"}

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug.log

@@ -0,0 +1,27 @@
+2025-03-27 17:04:44,257 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Current SDK version is 0.18.6
+2025-03-27 17:04:44,257 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Configure stats pid to 1469429
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Loading settings from /home/chyang/.config/wandb/settings
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Loading settings from /home/chyang/workspace/LLM-BC/wandb/settings
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'llmbc/workspace/train_mlp_projector_workspace.py', 'program_abspath': '/home/chyang/workspace/LLM-BC/llmbc/workspace/train_mlp_projector_workspace.py', 'program': '/home/chyang/workspace/LLM-BC/./llmbc/workspace/train_mlp_projector_workspace.py'}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_setup.py:_flush():79] Applying login settings: {}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:_log_setup():533] Logging user logs to /home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug.log
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:_log_setup():534] Logging internal logs to /home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/logs/debug-internal.log
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():619] calling init triggers
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'name': 'train_mlp_projector', '_target_': 'llmbc.workspace.train_mlp_projector_workspace.TrainMlpProjectorWorkspace', 'obs_dim': 16, 'action_dim': 2, 'task_name': 'blockpush_lowdim_seed', 'exp_name': 'default', 'model_name': 'meta-llama/Llama-3.2-1B-Instruct', 'horizon': 1, 'n_obs_steps': 1, 'n_action_steps': 1, 'n_latency_steps': 0, 'past_action_visible': False, 'llm_translator': {'_target_': 'llmbc.translator.llm_translator.LLMTranslator', 'cfg': {'name': 'meta-llama/Llama-3.2-1B-Instruct', 'model_name': 'Llama-3.2-1B-Instruct', 'use_quantization': False, 'load_from_checkpoint': False, 'adaptor_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100', 'use_orig_model': False, 'use_joint_mlp_projector': True, 'load_from_mlp_projector_checkpoint': False, 'mlp_projector_checkpoint_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin', 'max_length': 100, 'config_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig', 'causal_lm_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM', 'lora_config': {'r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'bias': 'none', 'task_type': 'CAUSAL_LM'}, 'prompter': {'_target_': 'llmbc.translator.prompter.llama_prompter.LlamaPrompter', 'use_joint_mlp_projector': True}, 'hydra': {'job': {'override_dirname': 'meta-llama/Llama-3.2-1B-Instruct'}, 'run': {'dir': 'data/outputs/2025.03.27/17.04.38_meta-llama/Llama-3.2-1B-Instruct'}}}, 'obs_dim': 16, 'action_dim': 2, 'horizon': 1, 'n_obs_steps': 1, 'n_action_steps': 1}, 'dataloader': {'batch_size': 8, 'num_workers': 0, 'shuffle': True, 'pin_memory': False, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 8, 'num_workers': 0, 'shuffle': True, 'pin_memory': False, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.Adam', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda', 'seed': 42, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 10, 'num_epochs': 10, 'gradient_accumulate_every': 64, 'grad_norm_clip': 1, 'enable_normalizer': True, 'checkpoint_every': 1, 'val_every': 1, 'sample_every': 1, 'sample_max_batch': 128, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'project': 'llm_module_training', 'resume': True, 'mode': 'online', 'name': '2025.03.27-17.04.43_train_mlp_projector_blockpush_lowdim_seed', 'tags': ['train_mlp_projector', 'blockpush_lowdim_seed', 'default'], 'id': None, 'group': None}, 'checkpoint': {'topk': {'monitor_key': 'val_loss', 'mode': 'min', 'k': 5, 'format_str': 'epoch={epoch:04d}-val_loss={val_loss:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed', 'wandb_name_base': '2025.03.27-17.04.43_train_mlp_projector_blockpush_lowdim_seed'}, 'task': {'name': 'blockpush_lowdim_seed', 'obs_dim': 16, 'action_dim': 2, 'keypoint_dim': 2, 'obs_eef_target': True, 'env_runner': {'_target_': 'llmbc.env_runner.blockpushing_lowdim_runner.BlockPushingLowdimRunner', 'env_name': 'llf-blockpushing-BlockPushMultimodal-v0', 'instruction_type': 'b', 'feedback_type': ['hp', 'hn', 'fp'], 'visual': False, 'max_steps': 350, 'n_obs_steps': 1, 'n_action_steps': 1}, 'dataset': {'_target_': 'llmbc.dataset.blockpush_lowdim_dataset.BlockPushLowdimDataset', 'data_path': 'datasets/BlockPushMultimodal-v0-general.pt', 'data_path2': 'datasets/BlockPushMultimodal-v0.pt', 'horizon': 1, 'pad_before': 0, 'pad_after': 0, 'obs_eef_target': True, 'use_manual_normalizer': False, 'seed': 42, 'val_ratio': 0.02, 'dummy_normalizer': False}, 'instructor': {'_target_': 'llmbc.translator.instructor.blockpush_instructor.BlockPushMultimodal_v0_instructor.BlockPushMultimodalV0Instructor'}}, 'llm': {'name': 'meta-llama/Llama-3.2-1B-Instruct', 'model_name': 'Llama-3.2-1B-Instruct', 'use_quantization': False, 'load_from_checkpoint': False, 'adaptor_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100', 'use_orig_model': False, 'use_joint_mlp_projector': True, 'load_from_mlp_projector_checkpoint': False, 'mlp_projector_checkpoint_path': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.14/13.31.49_train_llm_lowdim_sweep-v2/meta-llama/Llama-3.2-1B-Instruct-finetuned-sweep-v2/checkpoint-7100/mlp_projector.bin', 'max_length': 100, 'config_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaConfig', 'causal_lm_target': 'llmbc.model.llm.llama_lowdim_model.LowdimLlamaForCausalLM', 'lora_config': {'r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'bias': 'none', 'task_type': 'CAUSAL_LM'}, 'prompter': {'_target_': 'llmbc.translator.prompter.llama_prompter.LlamaPrompter', 'use_joint_mlp_projector': True}, 'hydra': {'job': {'override_dirname': 'meta-llama/Llama-3.2-1B-Instruct'}, 'run': {'dir': 'data/outputs/2025.03.27/17.04.38_meta-llama/Llama-3.2-1B-Instruct'}}}}
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():669] starting backend
+2025-03-27 17:04:44,258 INFO    MainThread:1469429 [wandb_init.py:init():673] sending inform_init request
+2025-03-27 17:04:44,259 INFO    MainThread:1469429 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-27 17:04:44,259 INFO    MainThread:1469429 [wandb_init.py:init():686] backend started and connected
+2025-03-27 17:04:44,264 INFO    MainThread:1469429 [wandb_init.py:init():781] updated telemetry
+2025-03-27 17:04:44,322 INFO    MainThread:1469429 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2025-03-27 17:04:44,978 INFO    MainThread:1469429 [wandb_init.py:init():867] starting run threads in backend
+2025-03-27 17:04:45,079 INFO    MainThread:1469429 [wandb_run.py:_console_start():2451] atexit reg
+2025-03-27 17:04:45,079 INFO    MainThread:1469429 [wandb_run.py:_redirect():2299] redirect: wrap_raw
+2025-03-27 17:04:45,081 INFO    MainThread:1469429 [wandb_run.py:_redirect():2364] Wrapping output streams.
+2025-03-27 17:04:45,081 INFO    MainThread:1469429 [wandb_run.py:_redirect():2389] Redirects installed.
+2025-03-27 17:04:45,082 INFO    MainThread:1469429 [wandb_init.py:init():911] run started, returning control to user process
+2025-03-27 17:04:45,083 INFO    MainThread:1469429 [wandb_run.py:_config_callback():1389] config_cb None None {'output_dir': '/home/chyang/workspace/LLM-BC/data/outputs/2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed'}
+2025-03-28 10:26:58,370 WARNING MsgRouterThr:1469429 [router.py:message_loop():75] message_loop has been closed

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/run-20250327_170444-0qs6crol/run-0qs6crol.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66aec1661440b055db48d2560924e71555af3599a76b4cc218984bf359d7366b
+size 194051744

2025.03.27/17.04.38_train_mlp_projector_blockpush_lowdim_seed/wandb/wandb-resume.json

@@ -0,0 +1 @@
+{"run_id": "0qs6crol"}