Question: else: sz = exp_max + 6 sz = max(min_sz, sz) <del> format = '{:' + str(sz) + '.4f}' </del> <ins> format = '{{:{}.{}f}}'.format(sz, prec) </ins> return format, scale, sz def _tensor_str(self): <ins> n = PRINT_OPTS.edgeitems has_hdots = self.size()[-1] > 2*n has_vdots = self.size()[-2] > 2*n print_full_mat = not has_hdots and not has_vdots formatter = _number_format(self, min_sz=3 if not print_full_mat else 0) print_dots = self.numel() >= PRINT_OPTS.threshold dim_sz = max(2, max(len(str(x)) for x in self.size())) dim_fmt = "{:^" + str(dim_sz) + "}" dot_fmt = u"{:^" + str(dim_sz+1) + "}" </ins> counter_dim = self.ndimension() - 2 counter = torch.LongStorage(counter_dim).fill_(0) <del> counter[0] = -1 </del> <ins> counter[counter.size()-1] = -1 </ins> finished = False strt = '' while True: <del> for i in _range(counter_dim): </del> <ins> nrestarted = [False for i in counter] nskipped = [False for i in counter] for i in _range(counter_dim - 1, -1, -1): </ins> counter[i] += 1 <ins> if print_dots and counter[i] == n and self.size(i) > 2*n: counter[i] = self.size(i) - n nskipped[i] = True </ins> if counter[i] == self.size(i): <del> if i == counter_dim-1: </del> <ins> if i == 0: </ins> finished = True counter[i] = 0 <ins> nrestarted[i] = True </ins> else: break if finished: break <ins> elif print_dots: if any(nskipped): for hdot in nskipped: strt += dot_fmt.format('...') if hdot  else dot_fmt.format('') strt += 'n' if any(nrestarted): strt += ' ' for vdot in nrestarted: strt += dot_fmt.format(u'u22EE' if vdot else '') strt += 'n' </ins> if strt != '': <del> strt += 'n' strt += '({},.,.) = n'.format(','.join(str(i) for i in counter)) submatrix = reduce(lambda t,i: t.select(0, i), counter, self) strt += _matrix_str(submatrix, ' ') </del> <ins> strt += 'n' strt += '({},.,.) = n'.format( ','.join(dim_fmt.format(i) for i in counter)) submatrix = reduce(lambda t, i: t.select(0, i), counter, self) strt += _matrix_str(submatrix, ' ', formatter, print_dots) </ins> return strt <del> def _matrix_str(self, indent=''): fmt, scale, sz = _number_format(self) nColumnPerLine = int(math.floor((80-len(indent))/(sz+1))) </del> <ins> def __repr_row(row, indent, fmt, scale, sz, truncate=None): if truncate is not None: dotfmt = " {:^5} " return (indent + ' '.join(fmt.format(val/scale) for val in row[:truncate]) + dotfmt.format('...') + ' '.join(fmt.format(val/scale) for val in row[-truncate:]) + 'n') else: return indent + ' '.join(fmt.format(val/scale) for val in row) + 'n' def _matrix_str(self, indent='', formatter=None, force_truncate=False): n = PRINT_OPTS.edgeitems has_hdots = self.size(1) > 2*n has_vdots = self.size(0) > 2*n print_full_mat = not has_hdots and not has_vdots if formatter is None: fmt, scale, sz = _number_format(self, min_sz=5 if not print_full_mat else 0) else: fmt, scale, sz = formatter nColumnPerLine = int(math.floor((PRINT_OPTS.linewidth-len(indent))/(sz+1))) </ins> strt = '' firstColumn = 0 <del> while firstColumn < self.size(1): lastColumn = min(firstColumn + nColumnPerLine - 1, self.size(1)-1) if nColumnPerLine < self.size(1): strt += 'n' if firstColumn != 1 else '' strt += 'Columns {} to {} n{}'.format(firstColumn, lastColumn, indent) </del> <ins> if not force_truncate and  (self.numel() < PRINT_OPTS.threshold or print_full_mat): while firstColumn < self.size(1): lastColumn = min(firstColumn + nColumnPerLine - 1, self.size(1)-1) if nColumnPerLine < self.size(1): strt += 'n' if firstColumn != 1 else '' strt += 'Columns {} to {} n{}'.format( firstColumn, lastColumn, indent) if scale != 1: strt += SCALE_FORMAT.format(scale) for l in _range(self.size(0)): strt += indent + (' ' if scale != 1 else '') row_slice = self[l, firstColumn:lastColumn+1] strt += ' '.join(fmt.format(val/scale) for val in row_slice) strt += 'n' firstColumn = lastColumn + 1 else: </ins> if scale != 1: strt += SCALE_FORMAT.format(scale) <del> for l in _range(self.size(0)): strt += indent + (' ' if scale != 1 else '') row_slice = self[l, firstColumn:lastColumn+1] strt += ' '.join(fmt.format(val/scale) for val in row_slice) + 'n' firstColumn = lastColumn + 1 </del> <ins> if has_vdots and has_hdots: vdotfmt = "{:^" + str((sz+1)*n-1) + "}" ddotfmt = u"{:^5}" for row in self[:n]: strt += __repr_row(row, indent, fmt, scale, sz, n) strt += indent + ' '.join([vdotfmt.format('...'), ddotfmt.format(u'u22F1'), vdotfmt.format('...')]) + "n" for row in self[-n:]: strt += __repr_row(row, indent, fmt, scale, sz, n) elif not has_vdots and has_hdots: for row in self: strt += __repr_row(row, indent, fmt, scale, sz, n) elif has_vdots and not has_hdots: vdotfmt = u"{:^" +  str(len(__repr_row(self[0], '', fmt, scale, sz))) +  "}n" for row in self[:n]: strt += __repr_row(row, indent, fmt, scale, sz) strt += vdotfmt.format(u'u22EE') for row in self[-n:]: strt += __repr_row(row, indent, fmt, scale, sz) else: for row in self: strt += __repr_row(row, indent, fmt, scale, sz) </ins> return strt def _vector_str(self): <del> fmt, scale, _ = _number_format(self) </del> <ins> fmt, scale, sz = _number_format(self) </ins> strt = '' ident = '' <ins> n = PRINT_OPTS.edgeitems dotfmt = u"{:^" + str(sz) + "}n" </ins> if scale != 1: strt += SCALE_FORMAT.format(scale) ident = ' ' <del> return strt + 'n'.join(ident + fmt.format(val/scale) for val in self) + 'n' </del> <ins> if self.numel() < PRINT_OPTS.threshold: return (strt + 'n'.join(ident + fmt.format(val/scale) for val in self) + 'n') else: return (strt + 'n'.join(ident + fmt.format(val/scale) for val in self[:n]) + 'n' + (ident + dotfmt.format(u"u22EE")) + 'n'.join(ident + fmt.format(val/scale) for val in self[-n:]) + 'n') </ins> def _str(self):

It seems that the following link that can be reached from here (check the screenshot) is broken: Page: Broken link: Screenshot: ! cc
this should be transferred to , but I don't have permissions to do that. Fix is available at pytorch/hub.


Question: The NVIDIA driver on your system is too old (found version {}). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx <del> Alternatively, go to: https://pytorch.org/binaries to install </del> <ins> Alternatively, go to: http://pytorch.org to install </ins> a PyTorch version that has been compiled with your version of the CUDA driver.""".format(str(torch._C._cuda_getDriverVersion()))) <ins> def _check_capability(): error_str = """ Found GPU%d %s which requires CUDA_VERSION >= %d for optimal performance and fast startup time, but your PyTorch was compiled with CUDA_VERSION %d. Please install the correct PyTorch binary using instructions from http://pytorch.org """ CUDA_VERSION = torch._C._cuda_getCompiledVersion() for d in range(device_count()): major = get_device_capability(d)[0] name = get_device_name(d) if CUDA_VERSION < 8000 and major >= 6: warnings.warn(error_str % (d, name, 8000, CUDA_VERSION)) elif CUDA_VERSION < 9000 and major >= 7: warnings.warn(error_str % (d, name, 8000, CUDA_VERSION)) </ins> def _lazy_call(callable): if _initialized: callable()

There's quite a few user-reports of installing the CUDA 7.5 binary of pytorch and then running on GTX 1080 / 1070, and there will be reports of installing 7.5 / 8.0 binaries and running on V100. Raising a user-warning in such situations and letting them know will fix the problem.
The version of CUDA used to compile the binary should be available thanks to this PR:
I think is a more robust way of getting the CUDA used to compile PyTorch, as currently if there are multiple CUDA installed in the machine, the heuristics used in might not match those used by CMake FindCUDA. I think that we should modify the to return the cuda version used during compilation. I have the impression that it will not require changing much because doesn't require cuda to execute. What do you think?


Question: "torch/csrc/cuda/Module.cpp", "torch/csrc/cuda/Storage.cpp", "torch/csrc/cuda/Stream.cpp", <del> "torch/csrc/cuda/Tensor.cpp", </del> "torch/csrc/cuda/AutoGPU.cpp", "torch/csrc/cuda/utils.cpp", "torch/csrc/cuda/serialization.cpp", <ins> *split_types("torch/csrc/cuda/Tensor.cpp"), </ins> ] if WITH_NCCL:

Trying to run on github actions via AWS instance gives me: I will of course set up now, but it shouldn't crash like that. May be it should use as a fallback? pt-2.0 cc
Is this a multitenant setup? Also curious which distro are you running, I was under the impression that most linux distros set home by default
this is AWS image. one default user. I think it's user, but it wants you to run as root, which is weird. I have manually now set to overcome this.
We should update triton to use pathlib or something


Question: pip_packages=pip_list_output, conda_packages=get_conda_packages(run_lambda), os=get_os(run_lambda), <ins> libc_version=get_libc_version(), </ins> gcc_version=get_gcc_version(run_lambda), clang_version=get_clang_version(run_lambda), cmake_version=get_cmake_version(run_lambda),

TensorPipe uses some advanced features of the Linux kernel. It does so on a "best effort" basis (i.e., use them if available, fall back on something else if not). This logic is unfortunately hard to test because it would require specific kernel versions on our CI machines. So it occasionally happens that we find out about issues when users report them. To help investigate these reports we've found it useful to know the kernel version and the glibc version used by the user (both when building and when running, if they're different). It would be great if these could be to the report produced by collect_env.
CC


Question: def cross(self, other, dim=-1): return Cross(dim)(self, other) <ins> def inverse(self): return Inverse.apply(self) </ins> def multinomial(self, num_samples=1, with_replacement=False): return Multinomial(num_samples, with_replacement)(self)

Command to install the latest nightly binary: installs : However, PyTorch itself reports : I guess this version tag might be defined during the build process from e.g. and then forwarded from , but also source builds should use based on the tag. Unsure, if this is just a reporting issue or if we are building the nightly binaries with and executing them with , so creating this issue to dig a bit further into it. CC cc
This is really weird, I've put up measure in place to validate against it... Will investigate today (and moving to pytorch repo)
can you please check that 2.2.0 reports (and is build) against the correct version of nccl library?
Confirmed for nightlies : Same issue for RC:
Looking at the build log Looks like we are compiling with: USESYSTEMNCCL
On Windows I see following exception: We compiling with should we return None rather then exception ?
In this case we are indeed using 2.18.1: Do you know why we are using the system-wide NCCL installation instead of the one? Should we delete the system NCCL and use by default?
Looks like its set here: Looks like it was by: Let me propose the PR to unset this flag : USESYSTEM_NCCL
I guess we are using the system NCCL installation to be able to during the runtime. If we would use the module I assume we would link NCCL into the PyTorch binaries. If so, we should make sure to update the NCCL version whenever is updated. This could break our dependency on the NCCL pip package.
Yes. Also when we update this nccl here:
We used following path  What would be good path for nccl 2.19.3 and cuda 12.1 ? As per this issue: Will be using 12.0 ncck 2.19.3
Installing system NCCL into the nightly build docker container feels great, but we need to figure out, how to keep it in sync with PyTorch. I wonder if one can, for example, just check nccl version from this requirements file and build from source?
yes probably we should build and upload nccl for our supported cuda versions, same way as we build docker containers, by following details here:
Verified with final rc


Question: if memo is None: memo = set() if self not in memo: <del> super(Container, self).modules(memo) </del> <ins> for m in super(Container, self).modules(memo): yield m </ins> for module in self.children(): for m in module.modules(memo): yield m

While using torch on a Macbook pro with M1 ship, using python 3.8.10 and torch 1.12.1, I came across a weird bug that happened because I forgot to put a tensor on MPS. The reproducible code is the following: As you can see, if is on MPS and on CPU, the line wrongly replaces the index in place of the index . I would expect to have an error saying both tensors should be on the same device instead. I get an error: cc
The error seems to happen somewhere within Also I'm getting a warning "The operator 'aten::nonzero' is not currently supported on the MPS backend and will fall back to run on the CPU. This may have performance implications" when printing tensors on MPS.
Hi - thanks for reporting this issue. This is actually an allowed usecase (copying CPU tensors into MPS tensors / MPS -CPU / MPS -MPS): The issue is on MPS side when copying a CPU tensor into a strided MPS tensor. I'll take a look.


Question: #include "THCTensorTypeUtils.cuh" #include "THCHalfAutoNumerics.cuh" #include "SharedMem.cuh" <ins> #include <algorithm> </ins> //////////////////////////////////////////////////////////////////////////////// // Spatial kernel (fast with large inner_size and small dim_size) ////////////////////////////////////////////////////////////////////////////////

mac 10.12.6 xcode 8.3.3 clang Apple LLVM version 8.0.0 (clang-800.0.42.1)
same here. Adding fixes it.
thanks! ! building now...
success!yeah! thanks again.


Question: def backward(ctx, grad_output): inverse, = ctx.saved_variables return -torch.mm(inverse.t(), torch.mm(grad_output, inverse.t())) <ins> class Gesv(Function): @staticmethod def forward(ctx, b, a): # TODO see if one can backprop through LU X, LU = torch.gesv(b, a) ctx.save_for_backward(X, a) ctx.mark_non_differentiable(LU) return X, LU @staticmethod def backward(ctx, grad_output, grad_LU=None): X, a = ctx.saved_variables grad_b, _ = torch.gesv(grad_output, a.t()) grad_a = -torch.mm(grad_b, X.t()) return grad_b, grad_a </ins>

There seems to be regression in the dynamo export of the MONAI VNet model as regards . Export script: The versions I'm using is torch 2.5.0.dev20240625+cu121 Previous torch versions such as were able to export this model successfully. Perhaps dynamo was generating FX graph without the expm1 node previously? cc: PyTorch version: 2.5.0.dev20240625+cu121 Is debug build: False CUDA used to build PyTorch: 12.1 ROCM used to build PyTorch: N/A OS: Ubuntu 22.04.4 LTS (x8664) GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 Clang version: Could not collect CMake version: version 3.29.0 Libc version: glibc-2.35 Python version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime) Python platform: Linux-6.5.0-35-generic-x8664-with-glibc2.35 Is CUDA available: True CUDA runtime version: 12.4.131 CUDAMODULELOADING set to: LAZY GPU models and configuration: GPU 0: NVIDIA RTX A5000 Nvidia driver version: 535.171.04 cuDNN version: Probably one of the following: /usr/lib/x8664-linux- /usr/lib/x8664-linux- /usr/lib/x8664-linux- /usr/lib/x8664-linux- /usr/lib/x8664-linux- /usr/lib/x8664-linux- /usr/lib/x8664-linux- /usr/lib/x8664-linux- HIP runtime version: N/A MIOpen runtime version: N/A Is XNNPACK available: True CPU: Architecture: x8664 CPU op-mode(s): 32-bit, 64-bit Address sizes: 46 bits physical, 48 bits virtual Byte Order: Little Endian CPU(s): 20 On-line CPU(s) list: 0-19 Vendor ID: GenuineIntel Model name: Intel(R) Core(TM) i9-9820X CPU @ 3.

30GHz CPU family: 6 Model: 85 Thread(s) per core: 2 Core(s) per socket: 10 Socket(s): 1 Stepping: 4 CPU max MHz: 4200.0000 CPU min MHz: 1200.0000 BogoMIPS: 6599.98 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constanttsc art archperfmon pebs bts repgood nopl xtopology nonstoptsc cpuid aperfmperf pni pclmulqdq dtes64 monitor dscpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse41 sse42 x2apic movbe popcnt tscdeadlinetimer aes xsave avx f16c rdrand lahflm abm 3dnowprefetch cpuidfault epb catl3 cdpl3 invpcidsingle pti ssbd mba ibrs ibpb stibp tprshadow flexpriority ept vpid eptad fsgsbase tscadjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdta avx512f avx512dq rdseed adx smap clflushopt clwb intelpt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqmllc cqmoccupllc cqmmbmtotal cqmmbmlocal dtherm ida arat pln pts hwp hwpactwindow hwpepp hwppkgreq vnmi mdclear flushl1d archcapabilities Virtualization: VT-x L1d cache: 320 KiB (10 instances) L1i cache: 320 KiB (10 instances) L2 cache: 10 MiB (10 instances) L3 cache: 16.5 MiB (1 instance) NUMA node(s): 1 NUMA node0 CPU(s): 0-19 Vulnerability Gather data sampling: Mitigation;

Microcode Vulnerability Itlb multihit: KVM: Mitigation: VMX disabled Vulnerability L1tf: Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable Vulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable Vulnerability Meltdown: Mitigation; PTI Vulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable Vulnerability Retbleed: Mitigation; IBRS Vulnerability Spec rstack overflow: Not affected Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and _user pointer sanitization Vulnerability Spectre v2: Mitigation; IBRS; IBPB conditional; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected Vulnerability Srbds: Not affected Vulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable Versions of relevant libraries: [pip3] numpy==1.24.4 [pip3] onnx==1.16.0 [pip3] onnxscript==0.1.0.dev20240625 [pip3] optree==0.11.0 [pip3] pytorch-quantization==2.1.2 [pip3] pytorch-triton==3.0.0+ [pip3] torch==2.5.0.dev20240625+cu121 [pip3] torch-tensorrt==2.3.0a0 [pip3] torchaudio==2.4.0.dev20240625+cu121 [pip3] torchdata==0.7.1a0 [pip3] torchtext==0.17.0a0 [pip3] torchvision==0.20.0.dev20240625+cu121 [conda] Could not collect
I am not very sure, but this looks like the kind of ops that should get decomp'ed. do you know? could you help take a look as part of the op support work you are doing? Thanks!
Yes, it looks like there is a graph-changed in dynamo, which dynamo now decomposes elu for some reasons. Using can preserve elu and then successfully export though.

Yes can run but we would need to be resolved first since the weights are still not exported properly.
I can take a look at this.
should fix this.

Here, I can't export trivial net with sort() using dynamoexport(). Legacy () works fine, dynamoexport errors out : Unsupported FX nodes: {'callfunction': ['']}. Pytorch nightly 06/03 cc
a missing op, could you take this?
Not sure if related:
Hmmm. Ideally, ONNX should add a sort op. It doesn't have one. I wonder what the legacy exporter converts it to.
should fix this issue.

Are there any plans to implement something like tensorflow's and ? exists but it doesn't have the backward step implemented.
One workaround for the moment would be to write as , which should support backprop since . Adding support for should be trivial, the gradient for it can be found in
I'll add backward support for , which should be equivalent to . I'll try to do it later today.
How difficult would it be to add in batched versions (where only the final two dimensions are square) of GESV and inverse? Are there any plans for this?
If we have batched version of , then adapting the autograd part is very simple. There is an open issue already that discuss implementing batched gesv
Actually, there is an open PR for that, so it might be coming soon
I'll rebase the batched version of gesv. It'll come in soon
May I ask why gesv only support . As I saw from , it support that A is a MxN matrix, and B is a NxK matrix, then solve a matrix X (NxK). If I want to use a non-square matrix A in gesv, would you mind to let me know how to implement in PyTorch?
this doesn't exactly answer your question but you can add zero-rows to the A matrix until it becomes square.
thanks a lot, I find the solution, can solve the problem.
Any plan to add gradient calculation for gels?
I am also wondering if will have differentiation support. My current workaround is to use instead
I actually implemented overdetermined case following tensorflow implementation. For backward pass I only did basic grad test. On Monday, March 18, 2019, Johnson Zhong <:
Thanks, seems to work fine!


Question: return grad_input <del> # TODO: gather </del> <ins> class Gather(Function): def __init__(self, dim): super(Gather, self).__init__() self.dim = dim def forward(self, input, index): assert not self.needs_input_grad[1], "Gather can't differentiate "  "the index" self.input_size = input.size() self.save_for_backward(index) return input.gather(self.dim, index) def backward(self, grad_output): index, = self.saved_tensors grad_input = grad_output.new(self.input_size).zero_() return grad_input.scatter_(self.dim, index, grad_output), None class Scatter(InplaceFunction): def __init__(self, dim, inplace=False): super(Scatter, self).__init__(inplace) self.dim = dim def forward(self, input, index, source): assert not self.needs_input_grad[1], "Scatter can't differentiate "  "the index" if self.inplace: self.mark_dirty(input) else: input = input.clone() self.save_for_backward(index) return input.scatter_(self.dim, index, source) def backward(self, grad_output): index, = self.saved_tensors grad_input = grad_source = None if self.needs_input_grad[0]: grad_input = grad_output.clone() grad_input.scatter_(self.dim, index, 0) if self.needs_input_grad[2]: grad_source = grad_output.gather(self.dim, index) return grad_input, None, grad_source </ins> # TODO: kthvalue # TODO: repeat # TODO: sort

<!-- A clear and concise description of what content in is an issue. If this has to do with the general website, please file an issue at instead. If this has to do with , please file an issue at --In the getting started section section the following quote appears: I guess this is from previous versions where CUDA support was not built-in and this can be removed, correct? cc
This seems to be easy fixable docs issue
If gets merged this is fixed.
Thanks for the PR Just merged it.


Question: self.assertEqual(a[()], a) self.assertEqual(a[()].data_ptr(), a.data_ptr()) <del> # @unittest.skip('failing') </del> def test_empty_fancy_index(self): # Empty list index creates an empty array a = tensor([1, 2, 3])

In PyTorch master 0.4 , variables cannot be indexed. This used to work in PyTorch 0.2 :
This is breaking some of Pyro's inference algorithms on PyTorch master:


Question: def _test_sparse_mask_hybrid_fixed(self): i = self.IndexTensor([ <del> [1, 3, 3, 0, 4], [2, 1, 1, 2, 3], </del> <ins> [1, 3, 0, 4], [2, 1, 2, 3], </ins> ]) <del> v = self.ValueTensor([[1, 2], [2, 3], [3, 4], [4, 5], [5, 6]]) x = self.SparseTensor(i, v, torch.Size([5, 4, 2])) </del> <ins> v = self.ValueTensor([[1, 2], [2, 3], [3, 4], [4, 5]]) # TODO: This is also testing that, if coalesce is a no-op, # the indices don't get permuted. I don't know if we actually # want to give this invariant. x = self.SparseTensor(i, v, torch.Size([5, 4, 2])).coalesce() </ins> dense = self.ValueTensor([ [[1, 3], [2, 2], [3, 3], [4, 2]], [[5, 7], [6, 7], [7, 9], [8, 9]],

and I talked about the intended semantics of sparsemask when given a sparse tensor as an argument, and we came to the following conclusions: with an uncoalesced sparse tensor does not make sense, full stop. Unlike values, where your update might be linear and work correctly on an uncoalesced sparse tensor, there isn't any good reason why you want to pluck out values from the dense tensor multiple times, it is basically always a bug. the (eventual) documentation for sparsemask, we will recommend calling coalesce before doing any operations involving sparsemask (or nonlinear updates to indices/values). should add a test to sparsemask which checks if the flag of the input tensor is true, and raise an error if it is not. This is a cheap safety check which will prevent people from forgetting to call coalesce (as in (2)). There is a performance consequence to calling coalesce every iteration of training. If the gradient is producing coalesced vectors, we can alleviate this problem by adding a preprocessing step to coalesce() which tests if the input is already coalesced, and bugs out in that case. However, I think this is out of scope for this particular ticket for now.


Question: # State initialization if len(state) == 0: state['step'] = 0 <del> state['square_avg'] = grad.new().resize_as_(grad).fill_(1) </del> <ins> state['square_avg'] = grad.new().resize_as_(grad).zero_() </ins> square_avg = state['square_avg'] alpha = group['alpha']

We should unify this with our normal list creating code for the reasonable cases cc
Similar issue to this, I believe:

The hub tests are broken on macOS. I'm seeing SSL certificate related errors when trying to hit GitHub. Run the macOS tests on CI with TestHub enabled. I tried installing the latest version of from conda-forge (which pulled in ) but this didn't fix the issue. cc
Looks like the same error I'm getting trying to use Ubuntu 18.04 with fresh install of conda and Pytorch 1.2
I think this will all be solved with .

Porting TH operators is essential for code simplicity and performance reasons. Porting guides and Q&A are available in umbrella issue: Feel free to add as a reviewer to get a prioritized review.

This is of particular use in list comprehensions: This should work:
I'll need to adapt list comprehensions to use Iterables. I'll take a look!

I haven't looked into this in depth but this looks fishy: More here: cc
is this still an issue in 1.3.1?
I can repro this on master:
The problem is: removes the parameters from the module's parameter list and makes them normal class attributes: applies only to parameters, not to the weight attributes.  (which is called from acts on which includes the value of the weight attributes and the bias parameters. So, it contains a mix of CUDA and CPU tensors.
This is my workaround for right now: This applies weight norm to an rnn module and overrides the function to deal with the error.
This appears to be fixed. do you want to verify?
I tested this on nightly and it worked for me, so closing.

As mentioned in title, I want to make sure if pytorch can deal with the situation that each input in a batch has different shapes and thus different computation graphs.
Please don't open duplicated issue for discussion. Context will be missed. Try to discuss about it here:

Be Careful What You Backpropagate: A Case For Linear Output Activations & Gradient Boosting I can work on this if this can be to pytorch. Please let me know. Thanks!
This should be fairly straightforward to add in user code using . Eventually it could be to core, but its utility should probably be validated in external repos first.


Question: self.assertLessEqual(params.data.dist(solution), initial_dist) <ins> def _test_rosenbrock_sparse(self, constructor): params_t = torch.Tensor([1.5, 1.5]) params = Variable(torch.Tensor([1.5, 1.5]), requires_grad=True) params_c = Variable(torch.Tensor([1.5, 1.5]), requires_grad=True) optimizer = constructor([params]) optimizer_c = constructor([params_c]) solution = torch.Tensor([1, 1]) initial_dist = params.data.dist(solution) def eval(params, sparse_grad, w): optimizer.zero_grad() loss = rosenbrock(params) loss.backward() # params.grad.data now has the computed gradient. Turn # it into a sparse tensor params.grad.data.copy_(drosenbrock(params.data)) # This is really goofy if w: i = torch.LongTensor([[0]]) v = torch.DoubleTensor([params.grad.data[0]]) else: i = torch.LongTensor([[1]]) v = torch.DoubleTensor([params.grad.data[1]]) x = sparse.DoubleTensor(i, v, torch.Size([2])) if sparse_grad: params.grad.data = x else: params.grad.data = x.to_dense() return loss for i in range(2000): w = torch.rand(1)[0] > 0.5 optimizer.step(functools.partial(eval, params, True, w)) optimizer_c.step(functools.partial(eval, params_c, False, w)) self.assertEqual(params.data, params_c.data) self.assertLessEqual(params.data.dist(solution), initial_dist) </ins> def _test_basic_cases_template(self, weight, bias, input, constructor): weight = Variable(weight, requires_grad=True) bias = Variable(bias, requires_grad=True)

and I talked about the intended semantics of sparsemask when given a sparse tensor as an argument, and we came to the following conclusions: with an uncoalesced sparse tensor does not make sense, full stop. Unlike values, where your update might be linear and work correctly on an uncoalesced sparse tensor, there isn't any good reason why you want to pluck out values from the dense tensor multiple times, it is basically always a bug. the (eventual) documentation for sparsemask, we will recommend calling coalesce before doing any operations involving sparsemask (or nonlinear updates to indices/values). should add a test to sparsemask which checks if the flag of the input tensor is true, and raise an error if it is not. This is a cheap safety check which will prevent people from forgetting to call coalesce (as in (2)). There is a performance consequence to calling coalesce every iteration of training. If the gradient is producing coalesced vectors, we can alleviate this problem by adding a preprocessing step to coalesce() which tests if the input is already coalesced, and bugs out in that case. However, I think this is out of scope for this particular ticket for now.


Question: example_value = tensor[invalid_value_mask.eq(0)][0] tensor[invalid_value_mask] = example_value if invalid_value_mask.any(): <del> min_sz = 3 </del> <ins> min_sz = max(min_sz, 3) </ins> int_mode = True # TODO: use fmod?

It seems that the following link that can be reached from here (check the screenshot) is broken: Page: Broken link: Screenshot: ! cc
this should be transferred to , but I don't have permissions to do that. Fix is available at pytorch/hub.


Question: toDeviceTensor<THCIndex_t, 1>(state, target), toDeviceTensor<real, 1>(state, output), weights ? THCTensor_(data)(state, weights) : NULL, <ins> n_classes, </ins> ignore_index); <ins> THCudaCheck(cudaGetLastError()); </ins> if (weights) { THCTensor_(free)(state, weights); }

Claim: When I use an invalid label with F.crossentropy with reduce=False, I don't receive an error message. I do receive an error message when reduce=True. Code: Result: No error message received. Expected error message: Context: I was using , accidentally used an invalid label, and got extremely large losses and extremely small losses, and inconsistently so. Environment: OS: Ubuntu 16.04 on Docker and host PyTorch version: 0.3.0.post4 How you installed PyTorch: conda Python version: 3.6.3


Question: ``` docker build -t pytorch . ``` <del> Alternatively, if you want a runtime image, build with ``` docker build -t pytorch . -f tools/docker/Dockerfile_runtime ``` and run with nvidia-docker: </del> <ins> Alternatively, if you want to use a runtime image, you can use the pre-built one from Docker Hub and run with nvidia-docker: </ins> ``` <del> nvidia-docker run --rm -ti --ipc=host pytorch </del> <ins> nvidia-docker run --rm -ti --ipc=host pytorch/pytorch:latest </ins> ``` Please note that PyTorch uses shared memory to share data between processes, so if torch multiprocessing is used (e.g. for multithreaded data loaders) the default shared memory segment size that container runs with is not enough, and you

Do you plan to publish an official entry on docker hub?
any update? I think it's an important enhancement.
I believe this would just be pushing the image from ? If so, I can add it to Docker Hub if you like?
Configure automated builds on Docker Hub
The dockerfile you referenced builds from current master, I don't know if that's desirable. There is a runtime dockerfile in tools/docker that installs current binary release, I'd think that's what should be hosted on the dockerhub.
yup, that makes more sense. Thanks
Oops, looks like runtime dockerfile would be broken now (see discussion in #beginner, conda packages conflict with conda numpy).
Just tested, the runtime dockerfile seems to work just fine. Pulls in numpy 1.13.1 which seems to not have any issues.
pre-built images are now available, you can run them with:


Question: return SymbolicCallArg(expr, tree.numel) def generate_workspace_allocation(self, nbytes, device, zero_fill): <ins> if isinstance(nbytes, sympy.Expr): nbytes = V.graph.sizevars.size_hint(nbytes) </ins> line = self.make_allocation( "workspace", device, torch.uint8, shape=(nbytes,), stride=(1,) )

Repro: Error: cc
Un-assign myself to let work on it.
, I can't reproduce the issue. The is no longer in the codebase. I gave a try of , which worked well. Could you double check if you can reproduce the issue?
Yes, I can still reproduce with your cmdline. Can you retry after deleting your local cache?


Question: T bern_uniform = bernoulli[idx]; int _mask = (int) THCNumerics<T>::lt(bern_uniform, q[rand_ind]); output[idx] = J[rand_ind]*(1 -_mask) + (rand_ind+1L) * _mask; <del> } </del> <ins> } </ins> } template <typename T>

Looking at the values returned from tn.multinomial(1) and counting them, it seems like it returns the values of the multinomial distribution taken from t, i.e. that it has ignored the signs of the values. However, it seem like always returns a tensor filled with zeros.
looking into this


Question: (e.g. BatchNorm stats) are broadcast form the module in process of rank 0, to all other replicas in the system in every iteration. <ins> .. warning:: Forward and backwrad hooks defined on :attr:`module` and its submodules won't be invoked anymore, unless the hooks are initialized in the :meth:`forward` method. </ins> Args: module: module to be parallelized device_ids: CUDA devices (default: all devices)

PyTorch version: 0.3.0 Python version: 3.5 GPU models and configuration: 8x NVIDIA Titan I found that requiresgrad setting in module is ignored when module is wrapped with nn.DataParallel. For example, This code should be executed because very small part of network activation (only fc layer) is stored for backward computation, but it results to run-time error: Internally, I found a crack in function which is in . In replicate function, it copies all parameter in module (# of replica times) with . In broadcasting code, it just defines new with default constructor parameter, which is always set to True. I think there can be some choices to fix this issue. is intended behavior for DataParallel, so we should use volatile (or ) to implement transfer learning in DataParallel model should be fixed to handle requires_grad when copying parameters. should be synchronizing in I think any of solution doesn't need lots of effort to fix it.


Question: include_dirs = [] extra_link_args = [] <del> extra_compile_args = ['-std=c++11'] </del> <ins> extra_compile_args = ['-std=c++11', '-Wno-write-strings'] </ins> cwd = os.path.dirname(os.path.abspath(__file__)) lib_path = os.path.join(cwd, "torch", "lib")

<!-- A clear and concise description of what the bug is. --The target name is used by a test executable in mkldnn. Steps to reproduce the behavior: BUILDTEST=1 build <!-- If you have a code sample, error messages, stack traces, please provide it here as well --<!-- A clear and concise description of what you expected to happen. --CMake configure passes. Please copy and paste the output from our (or fill out the checklist below manually). You can get the script and run it with: PyTorch Version (e.g., 1.0): v1.5.0-rc2 OS (e.g., Linux): Windows How you installed PyTorch (, , source): source Build command you used (if compiling from source): python build Python version: 3.x CUDA/cuDNN version: 9.2 / 10.1 / 10.2 / NA GPU models and configuration: Tesla M60 Any other relevant information: <!-- Add any other context about the problem here. --We should not build tests for mkldnn.
I have encountered the same error, how do I resolve it?
It is actually mkldnn that is out of sync. So running the following script should help.
Thanks a lot for this I posted this on the PyTorch Discussions but to no avail, and now I've finally understood the reason of this error. Thanks a lot. Surprisingly, for me the build works out if I build from the master repo, but changing to a different version always fails.


Question: THLongStorage_free(topKSize); #define RUN_K(INDEX_T, DIM, DIR)                                         <del> gatherTopK<real, INDEX_T, DIM, DIR>                                          </del> <ins> gatherTopK<real, INDEX_T, DIM, DIR>                                    </ins> <<<grid, block, 0, THCState_getCurrentStream(state)>>>(              inputInfo,                                                         sliceSize,

Repro: Expect two small tensors of small integers to be printed. Instead, only the first case works. In the second case, you get a series of failed assertions starting with this: I wonder if it is related to
is this still an outstanding issue? I cannot reproduce this with an install from the master branch.
The size which causes failure doesn't seem deterministic. Using a pretty recent build (a few days old), I now have to increase the "9" to a larger number to make it fail. Sometimes it will fail at 10, sometimes at 11. Here's an updated case that seems to fail reliably at some size, for me:


Question: for(int i = 0; i < input->nDimension; i++){ no_elements *= input->size[i]; } <del> no_elements *= scale_factor * scale_factor; </del> <ins> no_elements *= scale_factor; </ins> int d1; int d2;

Original post at: Original post content: code: Tested on Linux with pytorch 0.2.0+ & GTX 750 and Mac OS X Sierra with pytorch 0.2.0+ & GT 750M It seems to be work well with mode=‘linear’ and memory of gpu is larger than 1GB. I think it’s something wrong. Comment: successfully reproduced using the above snippet on master.


Question: y = x[2:] self.assertEqual(int(y), 3) <del> @unittest.skipIf(torch.set_flush_denormal(False), "flush_denormal not supported") </del> <ins> # skip this test for now as it affects all tests @unittest.skipIf(True, "flush_denormal not supported") </ins> def test_set_flush_denormal(self): tiny_float = 1e-42 tiny_double = 1e-320

hi, I just saw this error in the last few days. both for GPU and CPU only environment. Is the test precision assertion correct ?
On PPC64 it looks like denormal flags aren't being set correctly. This PR from 2 days ago denormals support, but likely assumed only AMD64 / x86
currently the setflushdenormal call will always return False if on ppc64le. The unittest has a skipIf as below. I believe the intention is to skip this test if not running on x86. But since setflushdenormal returns a False on ppc64le , then it actually runs the test and not run on x86 "flushdenormal not supported") I believe is you just add a not as below in the torch.setflushdenormal(False), "flush_denormal not supported") the test should then run on x86 but skipped for ppc64le or other platforms.
Doing calls that have side effects in skipIf is a bad idea. We should write a context manager that always resets the previous state + skips the test if the call returned False
what can we do in the meantime to skip over the denormal test, which is invalid, on ppc64 ?
Skip this test everywhere. These checks will affect all tests and we don’t want that
closed via


Question: THD_LIB = os.path.join(lib_path, 'libTHD.1.dylib') NCCL_LIB = os.path.join(lib_path, 'libnccl.1.dylib') <del> if WITH_NCCL and subprocess.call('ldconfig -p | grep libnccl >/dev/null', shell=True) == 0: </del> <ins> if WITH_NCCL and (subprocess.call('ldconfig -p | grep libnccl >/dev/null', shell=True) == 0 or subprocess.call('/sbin/ldconfig -p | grep libnccl >/dev/null', shell=True) == 0): </ins> SYSTEM_NCCL = True main_compile_args = ['-D_THP_CORE']

Hello, on Debian (and possibly Ubuntu and other systems), is in and not in non-root users' search paths. Thus this line: at does not do what it should when compiling not as root. Either writing or adding a second check with the directory would help building pytorch on these systems. Would such a change be acceptable? Thank you & best regards Thomas
Or fix this by exporting ?
such a chance would be absolutely acceptable.
fixed in PR


Question: THCAssertSameGPU(THCTensor_(checkGPU)(state, 2, self_, src_)); THArgCheck(src_->nDimension == 2, 1, "expected a matrix"); <del> THCTensor *src = src_; if (self_ == src_) src = THCTensor_(newContiguous)(state, src_); </del> <ins> if (self_ != src_) THCTensor_(resizeAs)(state, self_, src_); </ins> <del> int64_t stride0 = src->stride[0]; int64_t stride1 = src->stride[1]; real *start = THCTensor_(data)(state, src); </del> <ins> int64_t stride0 = self_->stride[0]; int64_t stride1 = self_->stride[1]; real *start = THCTensor_(data)(state, self_); </ins> TensorTriOp<real, 1> op(start, stride0, stride1, k); if (self_ == src_) { <del> if (!THC_pointwiseApply1(state, src, op)) { </del> <ins> if (!THC_pointwiseApply1(state, src_, op)) { </ins> THArgCheck(false, 2, CUTORCH_DIM_WARNING); } } else { <del> THCTensor_(resizeAs)(state, self_, src); if (!THC_pointwiseApply2(state, self_, src, op)) { </del> <ins> if (!THC_pointwiseApply2(state, self_, src_, op)) { </ins> THArgCheck(false, 2, CUTORCH_DIM_WARNING); } } <del> if (self_ == src_) THCTensor_(freeCopyTo)(state, src, src_); </del> THCudaCheck(cudaGetLastError()); }

On recent PyTorch master (built from source; 0.4.0a0+) on a P100, CUDA 9:
I think the results on CPU meets the defination of triu (upper triangular part of the matrix). It is really hard to handle with a tensor from expand operation with multi-thread not only on GPU but also on CPU. The unexpected result will also happen on CPU if we apply openmp to it. I think it is necessary to eleminate the difference between the implementation and defination.
It doesn’t matter if it’s cpu or not. triu doesn’t have an underscore which means it doesn’t operate in-place and should always give the same result (in new memory)


Question: self._data_ptrs = [] return <del> # This is quite ugly, but it allows us to reuse the cuDNN code without larger # modifications. It's really a low-level API that doesn't belong in here, but # let's make this exception. from torch.backends.cudnn import rnn from torch.backends import cudnn from torch.nn._functions.rnn import CudnnRNN handle = cudnn.get_handle() with warnings.catch_warnings(record=True): fn = CudnnRNN( self.mode, self.input_size, self.hidden_size, num_layers=self.num_layers, batch_first=self.batch_first, dropout=self.dropout, train=self.training, bidirectional=self.bidirectional, dropout_state=self.dropout_state, ) # Initialize descriptors fn.datatype = cudnn._typemap[any_param.type()] fn.x_descs = cudnn.descriptor(any_param.new(1, self.input_size), 1) fn.rnn_desc = rnn.init_rnn_descriptor(fn, handle) # Allocate buffer to hold the weights num_weights = rnn.get_num_weights(handle, fn.rnn_desc, fn.x_descs[0], fn.datatype) fn.weight_buf = any_param.new(num_weights).zero_() fn.w_desc = rnn.init_weight_descriptor(fn, fn.weight_buf) # Slice off views into weight_buf params = rnn.get_parameters(fn, handle, fn.weight_buf) all_weights = [[p.data for p in l] for l in self.all_weights] # Copy weights and update their storage rnn._copyParams(all_weights, params) for orig_layer_param, new_layer_param in zip(all_weights, params): for orig_param, new_param in zip(orig_layer_param, new_layer_param): orig_param.set_(new_param.view_as(orig_param)) self._data_ptrs = list(p.data.data_ptr() for p in self.parameters()) </del> <ins> with torch.cuda.device_of(any_param): # This is quite ugly, but it allows us to reuse the cuDNN code without larger # modifications. It's really a low-level API that doesn't belong in here, but # let's make this exception. from torch.backends.cudnn import rnn from torch.backends import cudnn from torch.nn._functions.rnn import CudnnRNN handle = cudnn.get_handle() with warnings.catch_warnings(record=True): fn = CudnnRNN( self.mode, self.input_size, self.hidden_size, num_layers=self.num_layers, batch_first=self.batch_first, dropout=self.dropout, train=self.training, bidirectional=self.bidirectional, dropout_state=self.dropout_state, ) # Initialize descriptors fn.datatype = cudnn._typemap[any_param.type()] fn.x_descs = cudnn.descriptor(any_param.new(1, self.input_size), 1) fn.rnn_desc = rnn.init_rnn_descriptor(fn, handle) # Allocate buffer to hold the weights self._param_buf_size = rnn.get_num_weights(handle, fn.rnn_desc, fn.x_descs[0], fn.datatype) fn.weight_buf = any_param.new(self._param_buf_size).zero_() fn.w_desc = rnn.init_weight_descriptor(fn, fn.weight_buf) # Slice off views into weight_buf params = rnn.get_parameters(fn, handle, fn.weight_buf) all_weights = [[p.data for p in l] for l in self.all_weights] # Copy weights and update their storage rnn._copyParams(all_weights, params) for orig_layer_param, new_layer_param in zip(all_weights, params): for orig_param, new_param in zip(orig_layer_param, new_layer_param): orig_param.set_(new_param.view_as(orig_param)) self._data_ptrs = list(p.data.data_ptr() for p in self.parameters()) </ins> def _apply(self, fn): ret = super(RNNBase, self)._apply(fn)

Since the new pytorch version, I have a warning when I that says: When I do as suggested, and then try to encode again, I get the following error though: Any idea?
do you have code the reproduces the issue?
What was the fix? I still see the error on Pytorch Version: 0.2.03 AttributeError: 'LSTM' object has no attribute 'parambufsize'


Question: other types will be a shallow copy and can be corrupted if written to in the model's forward pass. <ins> .. warning:: Forward and backwrad hooks defined on :attr:`module` and its submodules won't be invoked anymore, unless the hooks are initialized in the :meth:`forward` method. </ins> Args: module: module to be parallelized device_ids: CUDA devices (default: all devices)

PyTorch version: 0.3.0 Python version: 3.5 GPU models and configuration: 8x NVIDIA Titan I found that requiresgrad setting in module is ignored when module is wrapped with nn.DataParallel. For example, This code should be executed because very small part of network activation (only fc layer) is stored for backward computation, but it results to run-time error: Internally, I found a crack in function which is in . In replicate function, it copies all parameter in module (# of replica times) with . In broadcasting code, it just defines new with default constructor parameter, which is always set to True. I think there can be some choices to fix this issue. is intended behavior for DataParallel, so we should use volatile (or ) to implement transfer learning in DataParallel model should be fixed to handle requires_grad when copying parameters. should be synchronizing in I think any of solution doesn't need lots of effort to fix it.


Question: if (weight_dim != k) { std::stringstream ss; <del> ss << "Expected " << weight_dim << "-dimensional input for " << weight_dim << "-dimensional weight " << weight.sizes() << ", but got input of size " << input.sizes() << " instead"; </del> <ins> ss << "Expected " << k << "-dimensional weight for " << k << "-dimensional input " << input.sizes() << ", but got weight of size " << weight.sizes() << " instead"; </ins> throw std::runtime_error(ss.str()); } if (weight.size(0) < groups) {

OS: Ubuntu 16.04 PyTorch version: 0.3.0.post4 How you installed PyTorch (conda, pip, source): pip Python version: 3.5.2 CUDA/cuDNN version: CUDA 8 The below script produces an incorrect error message () The below script produces an incorrect error message ()


Question: (4.53, 3.83, -6.64, 2.06))).t() b = torch.Tensor(((8.58, 8.26, 8.48, -5.28), (9.35, -4.43, -0.70, -0.26))).t() <del> _test(a, b, expectedNorm) </del> <ins> _test_underdetermined(a, b, expectedNorm) </ins> # test overderemined expectedNorm = 17.390200628863

b = (600,1) A = (600,2) X, = (b,A) X.size() #torch.Size([2, 1]) Y, = (b.cuda(),A.cuda()) Y.size() #torch.Size([600, 1])
that is interesting. says that the first rows returned ( here) include the solution of the least-squares problem. I've verified that the first 2 rows of X and Y are the same so you can probably use that for now while we figure out what the rest of the output is.


Question: print('Distributed not available, skipping tests') sys.exit(0) <ins> SKIP_IF_NO_CUDA_EXIT_CODE = 75 def skip_if_no_cuda_distributed(func): func.skip_if_no_cuda_distributed = True @wraps(func) def wrapper(*args, **kwargs): if not torch.cuda.is_available(): sys.exit(SKIP_IF_NO_CUDA_EXIT_CODE) return func(*args, **kwargs) return wrapper </ins> @contextmanager def _lock():

is causing a crash when we try to build pytorch. Crash looks like this: (all times in US/West) Our 7AM, 8AM builds and earlier were all fine but 9AM, 10AM, 11AM builds all crashed with that same traceback. The new import was in which I think went in between 8AM and 9AM today See command below: See here if needed for : <!-- A clear and concise description of what you expected to happen. --Please copy and paste the output from our (or fill out the checklist below manually). You can get the script and run it with: PyTorch Version (e.g., 1.0): building from master OS (e.g., Linux): linux How you installed PyTorch (, , source): source Build command you used (if compiling from source): see above under 'reproduce' Python version: 3.6
maybe easier to read for build command used:
please try in the CircleCi environment?
in our build script seems to have fixed this. Thanks !


Question: log_scale = (self.temperature.new(self.temperature.shape).fill_(K).lgamma() - self.temperature.log().mul(-(K - 1))) score = logits - value.mul(self.temperature) <del> score = (score - log_sum_exp(score)).sum(-1) </del> <ins> score = (score - _log_sum_exp(score)).sum(-1) </ins> return score + log_scale

Leaving this issue here for higher visibility. Originally reported by here: Also mentioned by Piggybacking off of this issue, has strange behavior (reported here: )
I have fixed this locally. Could I send a PR?
Yes, go ahead!


Question: toDeviceTensor<real, 1>(state, gradOutput), toDeviceTensor<real, 2>(state, gradInput), weights ? THCTensor_(data)(state, weights) : NULL, <ins> n_classes, </ins> ignore_index); <ins> THCudaCheck(cudaGetLastError()); </ins> if (weights) { THCTensor_(free)(state, weights); }

Claim: When I use an invalid label with F.crossentropy with reduce=False, I don't receive an error message. I do receive an error message when reduce=True. Code: Result: No error message received. Expected error message: Context: I was using , accidentally used an invalid label, and got extremely large losses and extremely small losses, and inconsistently so. Environment: OS: Ubuntu 16.04 on Docker and host PyTorch version: 0.3.0.post4 How you installed PyTorch: conda Python version: 3.6.3


Question: if hasattr(torch, name) and name not in EXCLUDE_FUNCTIONAL: def fn(*inputs): output = getattr(torch, name)(*inputs) <del> if test_output_indices is None: return output else: return tuple(output[i] for i in test_output_indices) </del> <ins> return output_process_fn(output) </ins> f_args_variable = (self_variable,) + args_variable f_args_tensor = (self_tensor,) + args_tensor

I am having problem in running the backward propagation with svd. In particular, it seems working when the first dimension of the matrix x (below) is greater than the second. But I get the following error if the first is smaller than the second. PyTorch version: 2018.04.20 Is debug build: No CUDA used to build PyTorch: 8.0.61 OS: Ubuntu 16.04.4 LTS GCC version: (Ubuntu 5.4.0-6ubuntu1~16.04.9) 5.4.0 CMake version: Could not collect Python version: 3.5 Is CUDA available: No CUDA runtime version: No CUDA GPU models and configuration: No CUDA Nvidia driver version: No CUDA cuDNN version: No CUDA Versions of relevant libraries: [pip3] numpy (1.13.3) [pip3] torch (2018.4.20) [pip3] torchvision (0.2.0) [conda] pytorch 0.3.1 py35cuda8.0.61cudnn7.0.52 pytorch [conda] pytorch-nightly 2018.04.20 py35cuda8.0.61cudnn7.1.21 pytorch [conda] torchvision 0.2.0 py35heaa392f_1 pytorch
Repro'ed on 0.4 branch. works btw.
What does "Repro'ed" means? Should I install from that branch? I need to compute the kernel of a matrix, therefore I need the last columns of the matrix V (which is why I have to pass ).
It's a shorthand for "reproduced", which means that the bug still hasn't been fixed. Sorry, we'll try to get to it soon.


Question: IF(C_AVX2_FOUND) IF(MSVC) <del> SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX2.c PROPERTIES COMPILE_FLAGS "/Ox /arch:AVX2 ${C_AVX2_FLAGS}") </del> <ins> SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX2.c PROPERTIES COMPILE_FLAGS "${MSVC_OPT_FLAG}/arch:AVX2 ${C_AVX2_FLAGS}") </ins> ELSE(MSVC) SET_SOURCE_FILES_PROPERTIES(${PROJECT_SOURCE_DIR}/src/TH/vector/AVX2.c PROPERTIES COMPILE_FLAGS "-O3 ${C_AVX2_FLAGS}") ENDIF(MSVC)

The current master triggers the following error messages from Visual Studio 2015 (Debug build): 2------ Build started: Project: ATen, Configuration: Debug x64 ------ 2cl : Command line error D8016: '/Ox' and '/RTC1' command-line options are incompatible ... 14LINK : fatal error LNK1104: cannot open file '....srcATenDebug' It appears that they are triggered by *.c files in the ATen project, like "srcTHvectorAVX.c"
Why do you need Debug build?
Isn't ATen meant to support both Debug and Release builds? FYI I'm involved with the project at which does support both build configurations, and we would like to be able to link our project with ATen.
Well, you can remove the flags in as a workground.
We should patch this file to ignore this flag when building the library in debug mode. It shouldn't be too hard
I catched similar errors , when I Tryed to build Debug Version I try to build Release and catched next errors e:pytorchatensrcatencpuvec256vec256int.h(184): note: see reference to function template instantiation 'at::vec256::anonymous-namespace'::Vec256<int16t::blend<1(at::vec256::anonymous-namespace'::Vec256<int16t)' being compiled 28e:pytorchatensrcatencpuvec256vec256int.h(147): error C3861: 'mm256extract_epi16': identifier not found What I should do?
There're two issues in your description. The first question is easy. You'll also have to specify the flag to 1. The second question is about your compiler. I guess you are using VS2015 or earlier versions of VS2017. Make sure the version in the output is 19.11 after your enter in CMD.
Yes I am using VS2015. My Compiler is 19.0.2. Must I use VS 2017 in any case?
Sorry, but the answer is yes. The AVX2 functions was in VS2017 and they are used in PyTorch. So VS2017 19.11 is a must.
No problem:) Thank you.


Question: parameters[2] = make_param() param_list[2] = parameters[2] check() <ins> self.assertEqual(param_list[1:], nn.ParameterList(parameters[1:])) self.assertEqual(param_list[3:], nn.ParameterList(parameters[3:])) self.assertEqual(param_list[:-1], nn.ParameterList(parameters[:-1])) self.assertEqual(param_list[:-3], nn.ParameterList(parameters[:-3])) self.assertEqual(param_list[::-1], nn.ParameterList(parameters[::-1])) </ins> with self.assertRaises(TypeError): param_list += make_param()

it'd be great to pick out elements of of a sequential with array slicing:
I agree this idea!
However,
this has come up again (last in ). Shall we reconsider?
I attempted getting a to work. Is this fine? If this is fine, I can issue a PR too. Let me know what you think. I believe this modification needs to be done in the class.
cc: This issue has been open for quite sometime.


Question: .. autofunction:: linear <ins> :hidden:`bilinear` ~~~~~~~~~~~~~~~~ .. autofunction:: bilinear </ins> Dropout functions -----------------

is not available in the docs page, but the doc string has been in . This was mentioned earlier in

Oh, bilinear, I missed that last time I looked at it. I'll add a docstring.

returns nothing. Might be useful to have some linter in the build process to find Python functions missing

Pytorch 0.4.0 gives a segmentation fault with dilated 2D convolutions over a specific size. import torch import as nn model = nn.Conv2d(inchannels=64, outchannels=64, kernelsize=5, dilation=2) model(([1, 64, 1166, 1166])) # This runs model(([1, 64, 1167, 1167])) # This crashes PyTorch version: 0.4.0 Is debug build: No CUDA used to build PyTorch: 9.1.85 OS: CentOS Linux 7 (Core) GCC version: (GCC) 4.8.5 (Red Hat 4.8.5-28) CMake version: version 2.8.12.2 Python version: 3.6 Is CUDA available: Yes CUDA runtime version: 8.0.61 GPU models and configuration: GPU 0: GeForce GTX 1080 Ti GPU 1: TITAN X (Pascal) GPU 2: Quadro K2200 Nvidia driver version: 390.59 cuDNN version: Probably one of the following: /usr/local/cuda- /usr/local/cuda- /usr/local/cuda-7.5/lib64/libcudnnstatic.a /usr/local/cuda- /usr/local/cuda-8.0/lib64/libcudnn_static.a
do you think this is the same as the segfault you showed me a week ago on convolution?
ah yes. int overflow. I only fixed conv3d because I didn’t want to hurt perf. I’ll fix all of them.

This should be an easy one.. The function is implemented, just no sign of it in the docs.


Question: for (int i = 0; i < required_args; ++i) { if (!inputs[i].defined()) { std::stringstream ss; <del> ss << name << ": expected Variable at argument " << i << " (got None)"; </del> <ins> ss << name << ": expected Tensor at argument " << i << " (got None)"; </ins> throw std::runtime_error(ss.str()); } }

Considering the merge of and , and the "removal" of from the API, there are still some messages that reference . This might prove to be confusing for new users starting off from , as they probably might not have heard of . I did a code search to find out multiple instances of this in . PyTorch or Caffe2: PyTorch How you installed PyTorch (conda, pip, source): source (@ ) Build command you used (if compiling from source): OS: Ubuntu 16.04.2 PyTorch version: Python version: 3.5 CUDA/cuDNN version: 8.0.44 GPU models and configuration: GeForce 940M GCC version (if compiling from source): 5.4.0 CMake version: 3.5.1


Question: set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN") <del> find_path(CUDNN_INCLUDE_DIR cudnn.h </del> <ins> if($ENV{CUDNN_INCLUDE_DIR}) SET(CUDNN_INCLUDE_DIR $ENV{CUDNN_INCLUDE_DIR}) else($ENV{CUDNN_INCLUDE_DIR}) find_path(CUDNN_INCLUDE_DIR cudnn.h </ins> HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES cuda/include include) <ins> endif($ENV{CUDNN_INCLUDE_DIR}) </ins> <del> find_library(CUDNN_LIBRARY cudnn </del> <ins> if($ENV{CUDNN_LIBRARY}) SET(CUDNN_LIBRARY $ENV{CUDNN_LIBRARY}) else($ENV{CUDNN_LIBRARY}) find_library(CUDNN_LIBRARY cudnn </ins> HINTS ${CUDNN_LIB_DIR} ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) <ins> endif($ENV{CUDNN_LIBRARY}) </ins> find_package_handle_standard_args( CUDNN DEFAULT_MSG CUDNN_INCLUDE_DIR CUDNN_LIBRARY)

Traceback (most recent call last): File "", line 3, in <moduleimport model File "/home/abc/github/chinese-", line 5, in <modulefrom import crnnOcr File "/home/abc/github/chinese-", line 72, in <modulemodel,converter = crnnSource() File "/home/abc/github/chinese-", line 64, in crnnSource model = crnn.CRNN(32, 1, len(alphabet)+1, 256, 1).cuda() File "/usr/local/lib/python2.7/dist-", line 216, in cuda return (lambda t: t.cuda(device)) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 123, in apply self.flattenparameters() File "/usr/local/lib/python2.7/dist-", line 65, in flattenparameters if not or not (anyparam): File "/usr/local/lib/python2.7/dist-", line 82, in isacceptable if libcudnn() is None: File "/usr/local/lib/python2.7/dist-", line 58, in libcudnn 'but linked against {}'.format(compileversion, _cudnnversion)) RuntimeError: cuDNN version mismatch: PyTorch was compiled against 6021 but linked against 5110 How to solve the problem?
could you pull the latest master branch and recompile. I pushed a change yesterday that fixes this:
I remove the old code and pull new code, then compile, install.It still fails. /usr/local/lib/python2.7/dist- UserWarning: No training configuration found in save file: the model was not compiled. Compile it manually.

('No training configuration found in save file: ' Traceback (most recent call last): File "", line 3, in <moduleimport model File "/home/qianbb/github/chinese-", line 5, in <modulefrom import crnnOcr File "/home/qianbb/github/chinese-", line 72, in <modulemodel,converter = crnnSource() File "/home/qianbb/github/chinese-", line 64, in crnnSource model = crnn.CRNN(32, 1, len(alphabet)+1, 256, 1).cuda() File "/usr/local/lib/python2.7/dist-", line 216, in cuda return (lambda t: t.cuda(device)) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 100, in apply self.flattenparameters() File "/usr/local/lib/python2.7/dist-", line 66, in flattenparameters if not or not (anyparam): File "/usr/local/lib/python2.7/dist-", line 87, in isacceptable if libcudnn() is None: File "/usr/local/lib/python2.7/dist-", line 58, in libcudnn 'but linked against {}'.format(compileversion, _cudnnversion)) RuntimeError: cuDNN version mismatch: PyTorch was compiled against 6021 but linked against 5110 The library of cuda is shown as following screenshot. !
I found the issue, I fixed it in which we will merge shortly.
ThanksDoes it have been merged?
yes. it is merged.
It seems the fix doesn't work.

The log is shown as following Traceback (most recent call last): File "", line 3, in <moduleimport model File "/home/qianbb/github/chinese-", line 5, in <modulefrom import crnnOcr File "/home/qianbb/github/chinese-", line 72, in <modulemodel,converter = crnnSource() File "/home/qianbb/github/chinese-", line 64, in crnnSource model = crnn.CRNN(32, 1, len(alphabet)+1, 256, 1).cuda() File "/usr/local/lib/python2.7/dist-", line 222, in cuda return (lambda t: t.cuda(device)) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 100, in apply self.flattenparameters() File "/usr/local/lib/python2.7/dist-", line 66, in flattenparameters if not or not (anyparam): File "/usr/local/lib/python2.7/dist-", line 87, in isacceptable if libcudnn() is None: File "/usr/local/lib/python2.7/dist-", line 58, in libcudnn 'but linked against {}'.format(compileversion, _cudnnversion)) RuntimeError: cuDNN version mismatch: PyTorch was compiled against 6021 but linked against 5110
that is weird. Can I see the full log of in pytorch directory. Can you make sure you have latest commits with .
I am sure the code is up-to-date. First I uninstall the code using the command pip unisntall torch. Then I remove the old code using the command rm -r pytorch and git clone again. The log of install is
Sorry to trouble you. Is there any new information on this issue?
I spent some time resolving this issue and I don't see too much about it online. Try ensuring that your Pytorch build is correct for your specific versions of Cuda and Cudnn. While I had the correct version of Pytorch, I resolved the issue by installing a different build of the same version.

The build I installed is . Here are the list of versions and builds: You can download these files locally and install with conda: (assumes command is being run in a directory containing , where this file will be specific to your Cuda and Cudnn version.
feedback from this code PackagesNotFoundError
thank you - I have edited my comment to reflect that I installed from a local zip.


Question: if (cur_target == ignore_index) { continue; } <ins> assert(cur_target  >= 0 && cur_target  < n_classes); </ins> Dtype weight = weights ? weights[cur_target] : ScalarConvert<int, Dtype>::to(1); gradInput[index][cur_target] = -weight * gradOutput[index];

Claim: When I use an invalid label with F.crossentropy with reduce=False, I don't receive an error message. I do receive an error message when reduce=True. Code: Result: No error message received. Expected error message: Context: I was using , accidentally used an invalid label, and got extremely large losses and extremely small losses, and inconsistently so. Environment: OS: Ubuntu 16.04 on Docker and host PyTorch version: 0.3.0.post4 How you installed PyTorch: conda Python version: 3.6.3


Question: - THTensor* self - long k - CONSTANT __last_dim <del> - CONSTANT true - CONSTANT true </del> <ins> - arg: bool largest default: "true" kwarg_only: True - arg: bool sorted default: "true" kwarg_only: True </ins> - arguments: - arg: THTensor* values output: True

Hi.. I think this is a bug: So, it doesn't matter if descending is True or False, just specifying it explicitly requires dimension to be specified.. It would be better (I think) if descending could be specified alone..
Same issue for if is used:
sort was fixed in topk is still a problem.


Question: if arg['name'] in ['self', 'state', 'dataType', 'handle']: arg['ignore_check'] = True declaration['options'] = self.filter_unique_options(declaration['options']) <del> return declarations </del> <ins> return [d for d in declarations if not d.get('only_register', False)] </ins> def filter_unique_options(self, options): def signature(option):

We should check that the version from cudnn.h matches the version from the loaded shared library. Really weird things happen if they don't match.


Question: self.ceil_mode = ceil_mode def extra_repr(self): <del> return 'norm_type={norm_type}, kernel_size{kernel_size}, stride={stride}, '  </del> <ins> return 'norm_type={norm_type}, kernel_size={kernel_size}, stride={stride}, '  </ins> 'ceil_mode={ceil_mode}'.format(**self.__dict__)

is not available in the docs page, but the doc string has been in . This was mentioned earlier in

Oh, bilinear, I missed that last time I looked at it. I'll add a docstring.

returns nothing. Might be useful to have some linter in the build process to find Python functions missing

Pytorch 0.4.0 gives a segmentation fault with dilated 2D convolutions over a specific size. import torch import as nn model = nn.Conv2d(inchannels=64, outchannels=64, kernelsize=5, dilation=2) model(([1, 64, 1166, 1166])) # This runs model(([1, 64, 1167, 1167])) # This crashes PyTorch version: 0.4.0 Is debug build: No CUDA used to build PyTorch: 9.1.85 OS: CentOS Linux 7 (Core) GCC version: (GCC) 4.8.5 (Red Hat 4.8.5-28) CMake version: version 2.8.12.2 Python version: 3.6 Is CUDA available: Yes CUDA runtime version: 8.0.61 GPU models and configuration: GPU 0: GeForce GTX 1080 Ti GPU 1: TITAN X (Pascal) GPU 2: Quadro K2200 Nvidia driver version: 390.59 cuDNN version: Probably one of the following: /usr/local/cuda- /usr/local/cuda- /usr/local/cuda-7.5/lib64/libcudnnstatic.a /usr/local/cuda- /usr/local/cuda-8.0/lib64/libcudnn_static.a
do you think this is the same as the segfault you showed me a week ago on convolution?
ah yes. int overflow. I only fixed conv3d because I didn’t want to hurt perf. I’ll fix all of them.

This should be an easy one.. The function is implemented, just no sign of it in the docs.


Question: self.assertEqual(output[0][0].sum().data[0], 0) self.assertEqual(output[1][2].sum().data[0], 0) <ins> embedding = nn.Embedding(10, 20, padding_idx=0, sparse=True) input = Variable(torch.LongTensor([[0, 2, 4, 5], [4, 3, 0, 9]])) output = embedding(input) self.assertEqual(output[0][0].sum().data[0], 0) self.assertEqual(output[1][2].sum().data[0], 0) </ins> def test_embedding_max_norm(self): embedding = nn.Embedding(22, 5, max_norm=1.0) input = Variable(torch.LongTensor([2, 8, 8, 6]))

As reported here with repro. When sparse=True, paddingidx should probably error out (as the easiest solution). Alternatively, the gradients for paddingidx indices should be zeroed out explicitly in the sparse=True backward codepath.


Question: torch.randn(4, 4).cuda()) @unittest.skipIf(not TEST_MULTIGPU, "multi-GPU not supported") <ins> def test_broadcast_not_requiring_grad(self): variables = [ Variable(torch.randn(1, 2).cuda(), requires_grad=True), Variable(torch.randn(1, 2).cuda(), requires_grad=False), Variable(torch.randn(1, 2).cuda(), requires_grad=False), Variable(torch.randn(1, 2).cuda(), requires_grad=True), Variable(torch.randn(1, 2).cuda(), requires_grad=True), ] broadcasted_variables = Broadcast.apply((0, 1), *variables) for output_idx, broadcasted_var in enumerate(broadcasted_variables): input_var = variables[output_idx % len(variables)] self.assertEqual(input_var.requires_grad, broadcasted_var.requires_grad) @unittest.skipIf(not TEST_MULTIGPU, "multi-GPU not supported") </ins> def test_replicate(self): module = nn.Linear(10, 5).float().cuda() input = Variable(torch.randn(2, 10).float().cuda())

PyTorch version: 0.3.0 Python version: 3.5 GPU models and configuration: 8x NVIDIA Titan I found that requiresgrad setting in module is ignored when module is wrapped with nn.DataParallel. For example, This code should be executed because very small part of network activation (only fc layer) is stored for backward computation, but it results to run-time error: Internally, I found a crack in function which is in . In replicate function, it copies all parameter in module (# of replica times) with . In broadcasting code, it just defines new with default constructor parameter, which is always set to True. I think there can be some choices to fix this issue. is intended behavior for DataParallel, so we should use volatile (or ) to implement transfer learning in DataParallel model should be fixed to handle requires_grad when copying parameters. should be synchronizing in I think any of solution doesn't need lots of effort to fix it.


Question: "torch/csrc/autograd/functions/convolution.cpp", "torch/csrc/autograd/functions/init.cpp", "torch/csrc/nn/THNN_generic.cpp", <ins> *split_types("torch/csrc/Tensor.cpp") </ins> ] try:

Trying to run on github actions via AWS instance gives me: I will of course set up now, but it shouldn't crash like that. May be it should use as a fallback? pt-2.0 cc
Is this a multitenant setup? Also curious which distro are you running, I was under the impression that most linux distros set home by default
this is AWS image. one default user. I think it's user, but it wants you to run as root, which is weird. I have manually now set to overcome this.
We should update triton to use pathlib or something


Question: #include "torch/csrc/autograd/engine.h" #include "torch/csrc/autograd/functions/basic_ops.h" <ins> #include "torch/csrc/utils/auto_gpu.h" </ins> #include <atomic> #include <condition_variable>

Consider following code, and run it with multi gpus (e.g. 4): It will output: That is, the default device would always be zero even in DataParallel. My pytorch version is 0.1.122. I think this is not the desired behavior. It would cause some troubles. I tried to insert my own cuda kernel into backward to calculate the gradients, it become very slow, and I fixed it by (grad.getdevice()). Anyway, I think currentdevice in forward and backward should be the same, or could anyone explain to me why they are different?
this seems like something we should fix: ?
<!-- drci-comment-start --:pagefacingup: Preview :pagefacingup: Preview :question: Need help or want to give feedback on the CI? Visit our Note: Links to docs will display an error until the docs builds have been completed. This comment was automatically generated by Dr. CI and updates every 15 minutes. <!-- drci-comment-end --


Question: modules[2] = nn.Conv2d(5, 3, 2) module_list[2] = modules[2] check() <ins> self.assertEqual(module_list[1:], nn.ModuleList(modules[1:])) self.assertEqual(module_list[3:], nn.ModuleList(modules[3:])) self.assertEqual(module_list[:-1], nn.ModuleList(modules[:-1])) self.assertEqual(module_list[:-3], nn.ModuleList(modules[:-3])) self.assertEqual(module_list[::-1], nn.ModuleList(modules[::-1])) </ins> with self.assertRaises(TypeError): module_list += nn.ReLU()

it'd be great to pick out elements of of a sequential with array slicing:
I agree this idea!
However,
this has come up again (last in ). Shall we reconsider?
I attempted getting a to work. Is this fine? If this is fine, I can issue a PR too. Let me know what you think. I believe this modification needs to be done in the class.
cc: This issue has been open for quite sometime.


Question: -DNO_CUDA=%NO_CUDA% ^ -DNO_NNPACK=%NO_NNPACK% ^ -Dnanopb_BUILD_GENERATOR=0 ^ <del> -DCMAKE_BUILD_TYPE=Release </del> <ins> -DCMAKE_BUILD_TYPE=%BUILD_TYPE% </ins> %MAKE_COMMAND% cd ../..

The current master triggers the following error messages from Visual Studio 2015 (Debug build): 2------ Build started: Project: ATen, Configuration: Debug x64 ------ 2cl : Command line error D8016: '/Ox' and '/RTC1' command-line options are incompatible ... 14LINK : fatal error LNK1104: cannot open file '....srcATenDebug' It appears that they are triggered by *.c files in the ATen project, like "srcTHvectorAVX.c"
Why do you need Debug build?
Isn't ATen meant to support both Debug and Release builds? FYI I'm involved with the project at which does support both build configurations, and we would like to be able to link our project with ATen.
Well, you can remove the flags in as a workground.
We should patch this file to ignore this flag when building the library in debug mode. It shouldn't be too hard
I catched similar errors , when I Tryed to build Debug Version I try to build Release and catched next errors e:pytorchatensrcatencpuvec256vec256int.h(184): note: see reference to function template instantiation 'at::vec256::anonymous-namespace'::Vec256<int16t::blend<1(at::vec256::anonymous-namespace'::Vec256<int16t)' being compiled 28e:pytorchatensrcatencpuvec256vec256int.h(147): error C3861: 'mm256extract_epi16': identifier not found What I should do?
There're two issues in your description. The first question is easy. You'll also have to specify the flag to 1. The second question is about your compiler. I guess you are using VS2015 or earlier versions of VS2017. Make sure the version in the output is 19.11 after your enter in CMD.
Yes I am using VS2015. My Compiler is 19.0.2. Must I use VS 2017 in any case?
Sorry, but the answer is yes. The AVX2 functions was in VS2017 and they are used in PyTorch. So VS2017 19.11 is a must.
No problem:) Thank you.


Question: .. autofunction:: cumprod .. autofunction:: cumsum .. autofunction:: dist <ins> .. autofunction:: logsumexp </ins> .. autofunction:: mean .. autofunction:: median .. autofunction:: mode

Leaving this issue here for higher visibility. Originally reported by here: Also mentioned by Piggybacking off of this issue, has strange behavior (reported here: )
I have fixed this locally. Could I send a PR?
Yes, go ahead!


Question: { PyObject_GC_UnTrack(self); THPFunction_clear(self); <ins> self->cdata_ptr.~weak_ptr(); </ins> self->cdata.~PyFunction(); Py_TYPE(self)->tp_free((PyObject*)self); }

there is a bug. the code is from The code for installing cuda10.2 should be corrected to The code for installing cuda10.2 should be corrected to cc
Thank you for reporting the typo

This is how I implement the decoder of a sequence to sequence model I'm not sure about the new autograd mechanics but this worked in the previous version. If I didn't make the unrolling codes a function it will work. It will also work with CPU. I compiled from source with Python 2.7, CUDA 8.0 and Cudnn 6
Thanks for the minimal reproduction! It broke OpenNMT and a big internal model but we weren't looking forward to isolating what parts of those had the issue.
That's great, I'll take a look today. Thanks!
I've pushed a fix to .
How does the inference work in this case? I mean, once you train it how do you test it in the test phase?


Question: <ins> #pragma once // RAII structs to acquire and release Python's global interpreter lock (GIL) #include <Python.h> // Acquires the GIL on construction struct AutoGIL { AutoGIL() : gstate(PyGILState_Ensure()) { } ~AutoGIL() { PyGILState_Release(gstate); } PyGILState_STATE gstate; }; // Releases the GIL on construction struct AutoNoGIL { AutoNoGIL() : save(PyEval_SaveThread()) { } ~AutoNoGIL() { PyEval_RestoreThread(save); } PyThreadState* save; }; </ins>

Go to Look at the html. It contains the following: ! This problem manifests in google search. Searching for 1.8.1 docs brings up the "1.8.0 docs": ! We can probably fix this by doing a quick sed on the website but we should develop a solution for future releases. I think this has happened before in the past and am not sure what causes it. cc


Question: c = nn.Conv2d(3, 3, 3) self.assertRaises(RuntimeError, lambda: c(None)) <ins> def test_Conv2d_backward_twice(self): input = Variable(torch.randn(2, 3, 5, 5)) c = nn.Conv2d(3, 3, 3) o1 = c(input) o1.sum().backward() self.assertRaisesRegex(RuntimeError, 'Specify retain_variables=True', lambda: o1.sum().backward()) </ins> @unittest.skipIf(not TEST_CUDA, 'CUDA not available') def test_Conv2d_large_workspace(self): # These sizes require huge cuDNN workspaces. Make sure we choose a

Usually calling backwards twice gives this message: But conv2d just segfaults. $ pt A B Segmentation fault (core dumped)


Question: // For Convolution strategies that don't implicitly handle grad_bias, we add a helper // function here to perform it using simple Tensor operators static at::Tensor compute_grad_bias(const at::Tensor& grad_output) { <del> // grad_output is in N, C, H, W, we re-shape and reduce over spatial dims and batches </del> <ins> // grad_output is in N, C, H, W, we re-shape and reduce over spatial dims and batches </ins> return grad_output.contiguous().view({grad_output.size(0), grad_output.size(1), -1}).sum(0).sum(1); }

The following code: produces this error Also, removing the ReLU from the network: causes the error to change to: However, when the second convolution is removed as well: then the code runs error free I'm on a ppc64le architecture with a P100 GPU, CUDA 8.0 and PyTorch version '0.4.0a0+'
I'm also having this problem using branch v0.3.0
i could reproduce this as well. we'll look into this.
I'm having the same error :/
same error with gradient penalty for WGAN
When you execute , are you cause memory leak
no there shouldn't be a memory leak.
I am having an out of memory error as well, only when applying backward to the output of this method. I am using 3 titans GPUs so memory is not a problem I would assume. def computeGP(netD, realdata, realembed, fakedata, LAMBDA): BATCHSIZE = (0) alpha = (BATCHSIZE, 1) alpha = (BATCHSIZE, int(realdata.nelement() / BATCHSIZE)).contiguous().view(BATCHSIZE, 3, 64, 64) alpha = () interpolates = alpha realdata + ((1 - alpha) fakedata) interpolates = () interpolates = autograd.Variable(interpolates, requiresgrad=True) discinterpolates, = netD(interpolates, realembed) gradients = (outputs=discinterpolates, inputs=interpolates, (()).cuda(), creategraph=True, retaingraph=True, onlyinputs=True)[0] gradientpenalty = (((2, dim=1) - 1) * 2).mean() LAMBDA return gradient_penalty `
I am also using it with gradient penalty for WGAN, But When I use gradient_penalty.backward(), the memory leak is happen, My pytorch is 0.4.0a0+.
I can reproduce the memory leak with this: However the leak disappears when only one convolution is used: Also, if nn.Linear is used, there is no leak. Torch version: 0.4.0a0+
I've opened a separate issue for that memory leak:


Question: "torch/csrc/cuda/python_nccl.cpp", ] if WITH_CUDNN: <del> main_libraries += ['cudnn'] library_dirs.insert(0, CUDNN_LIB_DIR) </del> <ins> main_libraries += [CUDNN_LIBRARY] </ins> # NOTE: these are at the front, in case there's another cuDNN in CUDA path include_dirs.insert(0, CUDNN_INCLUDE_DIR) if not IS_WINDOWS:

Traceback (most recent call last): File "", line 3, in <moduleimport model File "/home/abc/github/chinese-", line 5, in <modulefrom import crnnOcr File "/home/abc/github/chinese-", line 72, in <modulemodel,converter = crnnSource() File "/home/abc/github/chinese-", line 64, in crnnSource model = crnn.CRNN(32, 1, len(alphabet)+1, 256, 1).cuda() File "/usr/local/lib/python2.7/dist-", line 216, in cuda return (lambda t: t.cuda(device)) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 123, in apply self.flattenparameters() File "/usr/local/lib/python2.7/dist-", line 65, in flattenparameters if not or not (anyparam): File "/usr/local/lib/python2.7/dist-", line 82, in isacceptable if libcudnn() is None: File "/usr/local/lib/python2.7/dist-", line 58, in libcudnn 'but linked against {}'.format(compileversion, _cudnnversion)) RuntimeError: cuDNN version mismatch: PyTorch was compiled against 6021 but linked against 5110 How to solve the problem?
could you pull the latest master branch and recompile. I pushed a change yesterday that fixes this:
I remove the old code and pull new code, then compile, install.It still fails. /usr/local/lib/python2.7/dist- UserWarning: No training configuration found in save file: the model was not compiled. Compile it manually.

('No training configuration found in save file: ' Traceback (most recent call last): File "", line 3, in <moduleimport model File "/home/qianbb/github/chinese-", line 5, in <modulefrom import crnnOcr File "/home/qianbb/github/chinese-", line 72, in <modulemodel,converter = crnnSource() File "/home/qianbb/github/chinese-", line 64, in crnnSource model = crnn.CRNN(32, 1, len(alphabet)+1, 256, 1).cuda() File "/usr/local/lib/python2.7/dist-", line 216, in cuda return (lambda t: t.cuda(device)) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 100, in apply self.flattenparameters() File "/usr/local/lib/python2.7/dist-", line 66, in flattenparameters if not or not (anyparam): File "/usr/local/lib/python2.7/dist-", line 87, in isacceptable if libcudnn() is None: File "/usr/local/lib/python2.7/dist-", line 58, in libcudnn 'but linked against {}'.format(compileversion, _cudnnversion)) RuntimeError: cuDNN version mismatch: PyTorch was compiled against 6021 but linked against 5110 The library of cuda is shown as following screenshot. !
I found the issue, I fixed it in which we will merge shortly.
ThanksDoes it have been merged?
yes. it is merged.
It seems the fix doesn't work.

The log is shown as following Traceback (most recent call last): File "", line 3, in <moduleimport model File "/home/qianbb/github/chinese-", line 5, in <modulefrom import crnnOcr File "/home/qianbb/github/chinese-", line 72, in <modulemodel,converter = crnnSource() File "/home/qianbb/github/chinese-", line 64, in crnnSource model = crnn.CRNN(32, 1, len(alphabet)+1, 256, 1).cuda() File "/usr/local/lib/python2.7/dist-", line 222, in cuda return (lambda t: t.cuda(device)) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 146, in apply (fn) File "/usr/local/lib/python2.7/dist-", line 100, in apply self.flattenparameters() File "/usr/local/lib/python2.7/dist-", line 66, in flattenparameters if not or not (anyparam): File "/usr/local/lib/python2.7/dist-", line 87, in isacceptable if libcudnn() is None: File "/usr/local/lib/python2.7/dist-", line 58, in libcudnn 'but linked against {}'.format(compileversion, _cudnnversion)) RuntimeError: cuDNN version mismatch: PyTorch was compiled against 6021 but linked against 5110
that is weird. Can I see the full log of in pytorch directory. Can you make sure you have latest commits with .
I am sure the code is up-to-date. First I uninstall the code using the command pip unisntall torch. Then I remove the old code using the command rm -r pytorch and git clone again. The log of install is
Sorry to trouble you. Is there any new information on this issue?
I spent some time resolving this issue and I don't see too much about it online. Try ensuring that your Pytorch build is correct for your specific versions of Cuda and Cudnn. While I had the correct version of Pytorch, I resolved the issue by installing a different build of the same version.

The build I installed is . Here are the list of versions and builds: You can download these files locally and install with conda: (assumes command is being run in a directory containing , where this file will be specific to your Cuda and Cudnn version.
feedback from this code PackagesNotFoundError
thank you - I have edited my comment to reflect that I installed from a local zip.


Question: #  binary_linux_libtorch_3.6m_cpu_test: #    environment: #      BUILD_ENVIRONMENT: "libtorch 3.6m cpu" <del> #    resource_class: gpu.medium </del> <ins> #    resource_class: gpu.nvidia.small </ins> #    <<: *binary_linux_test # #  binary_linux_libtorch_3.6m_cu90_test: #    environment: #      BUILD_ENVIRONMENT: "libtorch 3.6m cu90" <del> #    resource_class: gpu.medium </del> <ins> #    resource_class: gpu.nvidia.small </ins> #    <<: *binary_linux_test # docker_build_job:

As it still using deprecated resource class, see See example of the failure in
Followup: we need to plan better --we knew it was gonna happen and none of us fixed it early enough. Action item: go through tasks about deprecation in sprint planning


Question: } } <del> return F::detail::batch_norm( </del> <ins> return torch::nn::functional::detail::batch_norm( </ins> input, this->running_mean, this->running_var,

As part of pull request , F is redefined globally in nn/modules/batchnorm.h F is then used once in batchnorm.h and again in instancenorm.h not a bug, but a problem for anyone using F in some other way. it's solvable, but I don't see this pattern anywhere else in libtorch, was wondering if this use is an anomaly, and if we could use thanks cc
Thanks a lot for reporting! Yes this is an anomaly, and will be fixed by


Question: (Trace, (), ((S, S),)), (Cross, (), ((S, 3), (S, 3))), (Cross, (1,), ((S, 3, S), (S, 3, S)), 'dim'), <ins> (Inverse, (), ((S, S),), '', (), [skipIfNoLapack]), </ins> (Clone, (), ((S, M, S),)), (Squeeze, (), ((S, 1, M, 1),)), # TODO: enable neg dim checks

Command to install the latest nightly binary: installs : However, PyTorch itself reports : I guess this version tag might be defined during the build process from e.g. and then forwarded from , but also source builds should use based on the tag. Unsure, if this is just a reporting issue or if we are building the nightly binaries with and executing them with , so creating this issue to dig a bit further into it. CC cc
This is really weird, I've put up measure in place to validate against it... Will investigate today (and moving to pytorch repo)
can you please check that 2.2.0 reports (and is build) against the correct version of nccl library?
Confirmed for nightlies : Same issue for RC:
Looking at the build log Looks like we are compiling with: USESYSTEMNCCL
On Windows I see following exception: We compiling with should we return None rather then exception ?
In this case we are indeed using 2.18.1: Do you know why we are using the system-wide NCCL installation instead of the one? Should we delete the system NCCL and use by default?
Looks like its set here: Looks like it was by: Let me propose the PR to unset this flag : USESYSTEM_NCCL
I guess we are using the system NCCL installation to be able to during the runtime. If we would use the module I assume we would link NCCL into the PyTorch binaries. If so, we should make sure to update the NCCL version whenever is updated. This could break our dependency on the NCCL pip package.
Yes. Also when we update this nccl here:
We used following path  What would be good path for nccl 2.19.3 and cuda 12.1 ? As per this issue: Will be using 12.0 ncck 2.19.3
Installing system NCCL into the nightly build docker container feels great, but we need to figure out, how to keep it in sync with PyTorch. I wonder if one can, for example, just check nccl version from this requirements file and build from source?
yes probably we should build and upload nccl for our supported cuda versions, same way as we build docker containers, by following details here:
Verified with final rc


Question: gate_size = hidden_size self._all_weights = [] <del> self._param_buf_size = 0 </del> for layer in range(num_layers): for direction in range(num_directions): layer_input_size = input_size if layer == 0 else hidden_size * num_directions

Since the new pytorch version, I have a warning when I that says: When I do as suggested, and then try to encode again, I get the following error though: Any idea?
do you have code the reproduces the issue?
What was the fix? I still see the error on Pytorch Version: 0.2.03 AttributeError: 'LSTM' object has no attribute 'parambufsize'


Question: def backward(self, grad_output): tensors = self.saved_tensors input, params = tensors[0], tensors[1:] <del> grad_input = (input.new().resize_as_(input).zero_() </del> <ins> grad_input = (input.new().resize_as_(input) </ins> if self.needs_input_grad[0] else None,) grad_param = tuple(p.new().resize_as_(p).zero_() if self.needs_input_grad[i+1] else None for i, p in enumerate(params))

THNN convolutions are slower due to unnecessary .zero() calls on gradients: is not needed same for bias, prob worth adding a constant in C-side that controls whether to override grad or not
Same for batch_norm and auto thnn functions, not sure if all of them though. Prob the speedups will be quite significant.
I think they are needed. THNN functions accumulate the gradient instead of overwriting the buffer, don't they?
only grad wrt weight, inputs are overriden
We should reopen this, still many functions have unnecessary .zero() calls on gradinput and gradparam, e.g.: : : most of the functions don't need it
I think the ones are currently necessary.
for nn yes, but cudnn has: and on cudnnConvolutionBackwardFilter and on cudnnBatchNormalizationBackward which control addition of gradients and should be set to 1 and 0 in pytorch cudnn (it is and 1 in )
Yeah, we can remove it on BatchNorm. If you look at Conv, it already only zeros out grad_weight for cunn.
this is now fixed in master:


Question: } void ObjectPtrAllocator::free(void* ptr) { <del> object = nullptr; </del> <ins> { AutoGIL gil; object = nullptr; } </ins> allocator->free(allocatorContext, ptr); delete this; } void StorageWeakRefAllocator::free(void* ptr) { <del> PyGILState_STATE gstate = PyGILState_Ensure(); PyObject_SetAttrString(object.get(), "cdata", Py_None); object = nullptr; PyGILState_Release(gstate); </del> <ins> { AutoGIL gil; PyObject_SetAttrString(object.get(), "cdata", Py_None); object = nullptr; } </ins> allocator->free(allocatorContext, ptr); delete this; }

When both PyTorch and Pandas are imported, calling results in a segfault. The issue was originally reported by It looks like calling returns a null pointer here: Examining in GDB: I suspect it's both PyTorch and Pandas doing threading magic and somehow getting in each other's way. I haven't yet been able to provide a minimal example that triggers this behaviour and does not use my package; it's likely that I'm doing something odd that causes this. More documentation here: Possibly related issues: and Python version: 3.6 PyTorch version: v0.2.0 Pandas version: 0.20.1


Question: global Storage Tensor = _import_dotted_name(t) Storage = _import_dotted_name(t.replace('Tensor', 'Storage')) <ins> if 'cuda' in t: import torch.cuda torch.cuda.init() </ins> _C._set_default_tensor_type(Tensor)

When I was working on Vision and Audio CI, I often find that the dowloaded nightly PyTorch is CPU version instead of expected CUDA version. It will break the domain library CIs. It's often caused by nightly build failed or that day's nightly build isn't finished. There are 2 open issues for it. If this day's nightly PyTorch cuda build isn't available, I think it's better to provide last day's nightly builds rather than CPU version. cc
It looks that all nightly builds would be finished before 4 AM at PST, so vision CI failure probability might be higher in the morning of France.

Hi, I've updated pytorch to the lastest master () from github, but got an error when I use : What to do for workaround? Thanks! Jin
Here's a workaround. I think the problem is that pytorch hasn't initialized CUDA yet. At any rate, the following works for me:
Thanks but your workaround doesn't work for me. The error message still shows. Jin
Sorry, I meant the following: Could you please try if that works for you?
Yep. Just what I thought and was going to add comment here :smile: It works. Thank you! Jin


Question: <del> CMAKE_MINIMUM_REQUIRED(VERSION 3.0) </del> <ins> CMAKE_MINIMUM_REQUIRED(VERSION 2.8) </ins> SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) ################################################################################

Currently, CMake 3.0 or higher is required for THPP. This is annoying for building from source on Ubuntu 14.04 systems, as default CMake is 2.8, and updating it requires either compiling from source or adding some random repos:
it should work fine if you change it to 2.8. I'm testing this out in a sec, and will push it into master
Nope, it does not for me.


Question: } } <del> PyObject * THSPTensor_(NewEmpty)() { auto r = THSTensor_(new)(LIBRARY_STATE_NOARGS); THSPTensor_(initStorage)(r); return THSPTensor_(New)(r); } </del> PyObject * THSPTensor_(New)(THSTensor *tensor) { THSTensorPtr ptr(tensor);

This causes to not work properly on newly created empty sparse tensors. In the example below, both should print 1 instead.


Question: if hasattr(lib, 'cudnnGetErrorString'): lib.cudnnGetErrorString.restype = ctypes.c_char_p __cudnn_version = lib.cudnnGetVersion() <ins> compile_version = torch._C._cudnn_version() # Check that cuDNN major and minor versions match if (__cudnn_version // 100) != (compile_version // 100): raise RuntimeError( 'cuDNN version mismatch: PyTorch was compiled against {} ' 'but linked against {}'.format(compile_version, __cudnn_version)) </ins> else: lib = None return lib

We should check that the version from cudnn.h matches the version from the loaded shared library. Really weird things happen if they don't match.


Question: THPUtils_setError(error.c_str()); } <del> #if defined(NUMPY_TYPE_ENUM) || defined(THC_GENERIC_FILE) </del> <ins> #if defined(NUMPY_TYPE_ENUM) || (defined(WITH_NUMPY) && defined(THC_GENERIC_FILE)) </ins> #ifndef THC_REAL_IS_HALF #define load_real real

When I don't have a working numpy dependency detected (due to unrelated shenanigans), the build fails with this: And then a big pile of related errors. CC
This is most likely the problem:
CC
Ok, should probably read
Should I go ahead and make the change? I'll also remove the at line 193. The rest looks ok to me.
go for it.
Actually, the change I proposed is incorrect (I misread the second defined), but I'm on it.


Question: SparseTensor = getattr(torch.cuda.sparse, tensor_type) else: SparseTensor = getattr(torch.sparse, tensor_type) <ins> padding_idx = ctx.padding_idx indices = indices.view(1, -1) grad_output = grad_output.view(-1, ctx._weight_size[1]) if padding_idx is not None: nonpadding_indices_indices = (indices.view(-1) != padding_idx).nonzero().view(-1) indices = indices.index_select(1, nonpadding_indices_indices) grad_output = grad_output.index_select(0, nonpadding_indices_indices) </ins> grad_weight = SparseTensor( <del> indices.view(1, -1), grad_output.view(-1, ctx._weight_size[1]), </del> <ins> indices, grad_output, </ins> ctx._weight_size, ) return None, grad_weight, None, None, None, None, None

As reported here with repro. When sparse=True, paddingidx should probably error out (as the easiest solution). Alternatively, the gradients for paddingidx indices should be zeroed out explicitly in the sparse=True backward codepath.


Question: ensure_attr('ClassNLLCriterion', 'weights') ensure_attr('ParallelCriterion', 'repeatTarget') ensure_attr('MultiMarginCriterion', 'weights') <del> ensure_attr('SpatialConvolution', 'bias', 'finput', 'fgradInput', 'gradWeight', 'gradBias') </del> <ins> ensure_attr('SpatialConvolution', 'bias', 'finput', 'fgradInput', 'gradWeight', 'gradBias', '_gradOutput') </ins> attr_map('ReLU', {'val': 'value'}) attr_map('Threshold', {'val': 'value'}) attr_map('Unsqueeze', {'pos': 'dim'})

On Windows, torch.slogdet reproducibly fails when computing torch.slogdet on larger tensors. Here is the error: Steps to reproduce the behavior: Expected this op to succeed on Windows. This does not fail on Linux. cc
Which GPU card are you using on Windows?
I have an RTX 3080 and an RTX 3090 and see the issue on both. If this is a CUDA issue, could you point me to the right place to report it?
I can reproduce this with PyTorch 1.7.0 + RTX 3080.
It seems that the current implementation of this function is based on MAGMA. Will try to rebuild with newer versions.
It seems that the current windows MAGMA binaries are built without cuda arch 8.0 support. I will fix it.
Awesome! Thank you so much for the quick turnaround on this. Please let me know if you want me to test a fix.
Wait for the merge of


Question: return: self cname: resize cpu_half: True <ins> before_call: THPUtils_assert(arg_self->storage->flag & TH_STORAGE_RESIZABLE, "calling resize_ on a tensor that has non-resizable storage. Clone it first " "or create a new tensor instead."); </ins> arguments: - THTensor* self - arg: THSize* size

When both PyTorch and Pandas are imported, calling results in a segfault. The issue was originally reported by It looks like calling returns a null pointer here: Examining in GDB: I suspect it's both PyTorch and Pandas doing threading magic and somehow getting in each other's way. I haven't yet been able to provide a minimal example that triggers this behaviour and does not use my package; it's likely that I'm doing something odd that causes this. More documentation here: Possibly related issues: and Python version: 3.6 PyTorch version: v0.2.0 Pandas version: 0.20.1


Question: return type(self), (self.tolist(),) def __repr__(self): <del> return str(self) </del> <ins> return repr(str(self)) </ins> def __str__(self): <del> return _tensor_str._str(self) </del> <ins> # All strings are unicode in Python 3, while we have to encode unicode # strings in Python2. If we can't, let python decide the best # characters to replace unicode characters with. if sys.version_info > (3,): return _tensor_str._str(self) else: return _tensor_str._str(self).encode( sys.stdout.encoding or 'UTF-8', 'replace') </ins> def __iter__(self): return iter(map(lambda i: self.select(0, i), _range(self.size(0))))

It seems that the following link that can be reached from here (check the screenshot) is broken: Page: Broken link: Screenshot: ! cc
this should be transferred to , but I don't have permissions to do that. Fix is available at pytorch/hub.