/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:110: UserWarning: 'has_cuda' is deprecated, please use 'torch.backends.cuda.is_built()' torch.has_cuda, /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:111: UserWarning: 'has_cudnn' is deprecated, please use 'torch.backends.cudnn.is_available()' torch.has_cudnn, /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:117: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()' torch.has_mps, /data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/overrides.py:118: UserWarning: 'has_mkldnn' is deprecated, please use 'torch.backends.mkldnn.is_available()' torch.has_mkldnn, Traceback (most recent call last): File "isolated_nwm_infer.py", line 241, in main(args) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "isolated_nwm_infer.py", line 219, in main generate_time(args, curr_time_output_dir, idxs, model_lst, obs_image, gt_image, delta, secs, num_cond, device) File "isolated_nwm_infer.py", line 115, in generate_time x_pred_pixels = model_forward_wrapper(all_models, obs_image, curr_delta, timestep, args.latent_size, num_cond=num_cond, num_goals=1, device=device) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "isolated_nwm_infer.py", line 84, in model_forward_wrapper samples = diffusion.p_sample_loop( File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 456, in p_sample_loop for sample in self.p_sample_loop_progressive( File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 507, in p_sample_loop_progressive out = self.p_sample( File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 408, in p_sample out = self.p_mean_variance( File "/data1/tpz/nwm-main/diffusion/respace.py", line 98, in p_mean_variance return super().p_mean_variance(self._wrap_model(model), *args, **kwargs) File "/data1/tpz/nwm-main/diffusion/gaussian_diffusion.py", line 285, in p_mean_variance model_output = model(x, t, **model_kwargs) File "/data1/tpz/nwm-main/diffusion/respace.py", line 135, in __call__ return self.model(x, new_ts, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward else self._run_ddp_forward(*inputs, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward return self.module(*inputs, **kwargs) # type: ignore[index] File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 328, in _fn return fn(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 487, in catch_errors return hijacked_callback(frame, cache_entry, hooks, frame_state) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 641, in _convert_frame result = inner_convert(frame, cache_size, hooks, frame_state) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 133, in _fn return fn(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 389, in _convert_frame_assert return _compile( File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 569, in _compile guarded_code = compile_inner(code, one_graph, hooks, transform) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 189, in time_wrapper r = func(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 491, in compile_inner out_code = transform_code_object(code, transform) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/bytecode_transformation.py", line 1028, in transform_code_object transformations(instructions, code_options) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 458, in transform tracer.run() File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2069, in run super().run() File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 719, in run and self.step() File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 683, in step getattr(self, inst.opname)(inst) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2157, in RETURN_VALUE self.output.compile_subgraph( File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 833, in compile_subgraph self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/contextlib.py", line 75, in inner return func(*args, **kwds) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 957, in compile_and_call_fx_graph compiled_fn = self.call_user_compiler(gm) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 189, in time_wrapper r = func(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1024, in call_user_compiler raise BackendCompilerFailed(self.compiler_fn, e).with_traceback( File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1009, in call_user_compiler compiled_fn = compiler_fn(gm, self.example_inputs()) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/backends/distributed.py", line 436, in compile_fn submod_compiler.run(*example_inputs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/interpreter.py", line 138, in run self.env[node] = self.run_node(node) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_dynamo/backends/distributed.py", line 430, in run_node return curr_submod(*new_args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/graph_module.py", line 678, in call_wrapped return self._wrapped_call(self, *args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/graph_module.py", line 284, in __call__ raise e File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/fx/graph_module.py", line 274, in __call__ return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl return forward_call(*args, **kwargs) File ".1386", line 36, in forward File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/utils/_stats.py", line 20, in wrapper return fn(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_subclasses/fake_tensor.py", line 1250, in __torch_dispatch__ return self.dispatch(func, types, args, kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_subclasses/fake_tensor.py", line 1541, in dispatch r = func(*args, **kwargs) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_ops.py", line 448, in __call__ return self._op(*args, **kwargs or {}) File "/data1/tpz/anaconda3/envs/nwm2/lib/python3.8/site-packages/torch/_meta_registrations.py", line 4773, in meta__scaled_dot_product_flash query_reshaped = query_t.reshape(Nnz_q, num_heads, head_dim) torch._dynamo.exc.BackendCompilerFailed: backend='compile_fn' raised: RuntimeError: aten/src/ATen/RegisterMeta.cpp:7661: SymIntArrayRef expected to contain only concrete integers While executing %submod_1 : [num_users=10] = call_module[target=compiled_submod_1](args = (%getitem, %getitem_1), kwargs = {}) Original traceback: None Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information You can suppress this exception and fall back to eager by setting: import torch._dynamo torch._dynamo.config.suppress_errors = True