| /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:110: UserWarning: 'has_cuda' is deprecated, please use 'torch.backends.cuda.is_built()' |
| torch.has_cuda, |
| /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:111: UserWarning: 'has_cudnn' is deprecated, please use 'torch.backends.cudnn.is_available()' |
| torch.has_cudnn, |
| /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:117: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()' |
| torch.has_mps, |
| /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:118: UserWarning: 'has_mkldnn' is deprecated, please use 'torch.backends.mkldnn.is_available()' |
| torch.has_mkldnn, |
| Traceback (most recent call last): |
| File "isolated_nwm_infer.py", line 241, in <module> |
| main(args) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context |
| return func(*args, **kwargs) |
| File "isolated_nwm_infer.py", line 219, in main |
| generate_time(args, curr_time_output_dir, idxs, model_lst, obs_image, gt_image, delta, secs, num_cond, device) |
| File "isolated_nwm_infer.py", line 115, in generate_time |
| x_pred_pixels = model_forward_wrapper(all_models, obs_image, curr_delta, timestep, args.latent_size, num_cond=num_cond, num_goals=1, device=device) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context |
| return func(*args, **kwargs) |
| File "isolated_nwm_infer.py", line 84, in model_forward_wrapper |
| samples = diffusion.p_sample_loop( |
| File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 456, in p_sample_loop |
| for sample in self.p_sample_loop_progressive( |
| File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 507, in p_sample_loop_progressive |
| out = self.p_sample( |
| File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 408, in p_sample |
| out = self.p_mean_variance( |
| File "/data1/tpz/nwm-main/diffusion/respace.py", line 98, in p_mean_variance |
| return super().p_mean_variance(self._wrap_model(model), *args, **kwargs) |
| File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 285, in p_mean_variance |
| model_output = model(x, t, **model_kwargs) |
| File "/data1/tpz/nwm-main/diffusion/respace.py", line 135, in __call__ |
| return self.model(x, new_ts, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward |
| else self._run_ddp_forward(*inputs, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward |
| return self.module(*inputs, **kwargs) # type: ignore[index] |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
| return forward_call(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 328, in _fn |
| return fn(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
| return forward_call(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 487, in catch_errors |
| return hijacked_callback(frame, cache_entry, hooks, frame_state) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 641, in _convert_frame |
| result = inner_convert(frame, cache_size, hooks, frame_state) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 133, in _fn |
| return fn(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 389, in _convert_frame_assert |
| return _compile( |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 569, in _compile |
| guarded_code = compile_inner(code, one_graph, hooks, transform) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 189, in time_wrapper |
| r = func(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 491, in compile_inner |
| out_code = transform_code_object(code, transform) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/bytecode_transformation.py", line 1028, in transform_code_object |
| transformations(instructions, code_options) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 458, in transform |
| tracer.run() |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2069, in run |
| super().run() |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 719, in run |
| and self.step() |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 683, in step |
| getattr(self, inst.opname)(inst) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2157, in RETURN_VALUE |
| self.output.compile_subgraph( |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 833, in compile_subgraph |
| self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/contextlib.py", line 75, in inner |
| return func(*args, **kwds) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 957, in compile_and_call_fx_graph |
| compiled_fn = self.call_user_compiler(gm) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 189, in time_wrapper |
| r = func(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1024, in call_user_compiler |
| raise BackendCompilerFailed(self.compiler_fn, e).with_traceback( |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1009, in call_user_compiler |
| compiled_fn = compiler_fn(gm, self.example_inputs()) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/backends/distributed.py", line 436, in compile_fn |
| submod_compiler.run(*example_inputs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/interpreter.py", line 138, in run |
| self.env[node] = self.run_node(node) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/backends/distributed.py", line 430, in run_node |
| return curr_submod(*new_args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/graph_module.py", line 678, in call_wrapped |
| return self._wrapped_call(self, *args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/graph_module.py", line 284, in __call__ |
| raise e |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/graph_module.py", line 274, in __call__ |
| return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl |
| return forward_call(*args, **kwargs) |
| File "<eval_with_key>.1386", line 36, in forward |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/utils/_stats.py", line 20, in wrapper |
| return fn(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_subclasses/fake_tensor.py", line 1250, in __torch_dispatch__ |
| return self.dispatch(func, types, args, kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_subclasses/fake_tensor.py", line 1541, in dispatch |
| r = func(*args, **kwargs) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_ops.py", line 448, in __call__ |
| return self._op(*args, **kwargs or {}) |
| File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_meta_registrations.py", line 4773, in meta__scaled_dot_product_flash |
| query_reshaped = query_t.reshape(Nnz_q, num_heads, head_dim) |
| torch._dynamo.exc.BackendCompilerFailed: backend='compile_fn' raised: |
| RuntimeError: aten/src/ATen/RegisterMeta.cpp:7661: SymIntArrayRef expected to contain only concrete integers |
|
|
| While executing %submod_1 : [num_users=10] = call_module[target=compiled_submod_1](args = (%getitem, %getitem_1), kwargs = {}) |
| Original traceback: |
| None |
|
|
| Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information |
|
|
|
|
| You can suppress this exception and fall back to eager by setting: |
| import torch._dynamo |
| torch._dynamo.config.suppress_errors = True |
|
|
|
|
