File size: 21,511 Bytes
<!DOCTYPE html>
<meta charset="utf-8">

<html>

<style type="text/css">
body {
	font-family: "Titillium Web", "HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif;
	font-weight: 300;
	font-size: 17px;
	margin-left: auto;
	margin-right: auto;
	width: 980px;
}
h1 {
	font-weight:300;
	line-height: 1.15em;
}

h2 {
	font-size: 1.75em;
}
a:link,a:visited {
	color: #1367a7;
	text-decoration: none;
}
a:hover {
	color: #208799;
}
h1, h2, h3 {
	text-align: center;
}
h1 {
	font-size: 40px;
	font-weight: 500;
}
h2, h3 {
	font-weight: 400;
	margin: 16px 0px 4px 0px;
}
.paper-title {
	padding: 16px 0px 16px 0px;
}
section {
	margin: 32px 0px 32px 0px;
	text-align: justify;
	clear: both;
}
.col-5 {
	 width: 20%;
	 float: left;
}
.col-4 {
	 width: 25%;
	 float: left;
}
.col-2 {
	 width: 50%;
	 float: left;
}
.row, .author-row, .affil-row {
	 overflow: auto;
}
.author-row, .affil-row {
	font-size: 20px;
}
.row {
	margin: 16px 0px 16px 0px;
}
.authors {
	font-size: 18px;
}
.affil-row {
	margin-top: 16px;
}
.teaser {
	max-width: 100%;
}
.text-center {
	text-align: center;
}
.screenshot {
	width: 256px;
	border: 1px solid #ddd;
}
.screenshot-el {
	margin-bottom: 16px;
}
hr {
	height: 1px;
	border: 0;
	border-top: 1px solid #ddd;
	margin: 0;
}
.material-icons {
	vertical-align: -6px;
}
p {
	line-height: 1.25em;
}
.caption_justify {
	font-size: 16px;
	/*font-style: italic;*/
	color: #666;
	text-align: justify;
	margin-top: 0px;
	margin-bottom: 64px;
}
.caption {
	font-size: 16px;
	/*font-style: italic;*/
	color: #666;
	text-align: center;
	margin-top: 8px;
	margin-bottom: 64px;
}
.caption_inline {
	font-size: 16px;
	/*font-style: italic;*/
	color: #666;
	text-align: center;
	margin-top: 8px;
	margin-bottom: 0px;
}
.caption_bold {
	font-size: 16px;
	/*font-style: italic;*/
	color: #666;
	text-align: center;
	margin-top: 0px;
	margin-bottom: 0px;
	font-weight: bold;
}
video {
	display: block;
	margin: auto;
}
figure {
	display: block;
	margin: auto;
	margin-top: 10px;
	margin-bottom: 10px;
}
figure {
	display: block;
	margin: auto;
	margin-top: 10px;
	margin-bottom: 10px;
}
#bibtex pre {
	font-size: 14px;
	background-color: #eee;
	padding: 16px;
}
.blue {
	color: #2c82c9;
	font-weight: bold;
}
.orange {
	color: #d35400;
	font-weight: bold;
}
.flex-row {
	display: flex;
	flex-flow: row wrap;
	justify-content: space-around;
	padding: 0;
	margin: 0;
	list-style: none;
}
.paper-btn {
  position: relative;
  text-align: center;

  display: inline-block;
  margin: 8px;
  padding: 8px 18px;

  border-width: 0;
  outline: none;
  border-radius: 2px;

  background-color: #1367a7;
  color: #ecf0f1 !important;
  font-size: 20px;
  width: auto;
  height: auto;
  font-weight: 600;
}
.paper-btn-parent {
	display: flex;
	justify-content: center;
	margin: 16px 0px;
}
.paper-btn:hover {
	opacity: 0.85;
}
.container {
	margin-left: auto;
	margin-right: auto;
	padding-left: 16px;
	padding-right: 16px;
}
.venue {
	color: #1367a7;
}
</style>
<link href='https://fonts.googleapis.com/css?family=Titillium+Web:400,600,400italic,600italic,300,300italic' rel='stylesheet' type='text/css'>
<head>
	<title>Instant Neural Graphics Primitives with a Multiresolution Hash Encoding</title>
	<meta property="og:description" content="Instant Neural Graphics Primitives with a Multiresolution Hash Encoding"/>
	<link href="https://fonts.googleapis.com/css2?family=Material+Icons" rel="stylesheet">

	<meta name="twitter:card" content="summary_large_image">
	<meta name="twitter:creator" content="@mmalex">
	<meta name="twitter:title" content="Instant Neural Graphics Primitives with a Multiresolution Hash Encoding">
	<meta name="twitter:description" content="A paper from NVIDIA Research which presents a method for instant training & rendering of high-quality neural graphics primitives.">
	<meta name="twitter:image" content="https://nvlabs.github.io/instant-ngp/assets/twitter.jpg">
</head>

<body>
<div class="container">
	<div class="paper-title">
		<h1>Instant Neural Graphics Primitives with a Multiresolution Hash Encoding</h1>
	</div>

	<div id="authors">
		<div class="author-row">
			<div class="col-4 text-center"><a href="https://tom94.net">Thomas Müller</a></div>
			<div class="col-4 text-center"><a href="https://research.nvidia.com/person/alex-evans">Alex Evans</a></div>
			<div class="col-4 text-center"><a href="https://research.nvidia.com/person/christoph-schied">Christoph Schied</a></div>
			<div class="col-4 text-center"><a href="https://research.nvidia.com/person/alex-keller">Alexander Keller</a></div>
		</div>

		<div class="affil-row">
			<div class="col-1 text-center">NVIDIA</div>
		</div>

		<div class="affil-row">
			<div class="venue text-center"><b><a href="https://s2022.siggraph.org">ACM Transactions on Graphics (SIGGRAPH 2022)</a></b></div>
		</div>

		<div style="clear: both">
			<div class="paper-btn-parent">
				<a class="paper-btn" href="assets/mueller2022instant.pdf">
					<span class="material-icons"> description </span>
					Paper
				</a>
				<a class="paper-btn" href="assets/mueller2022instant.mp4">
					<span class="material-icons"> videocam </span>
					Video
				</a>
				<a class="paper-btn" href="https://tom94.net/data/publications/mueller22instant/mueller22instant-gtc.mp4">
					<span class="material-icons"> videocam </span>
					Presentation
				</a>
				<a class="paper-btn" href="https://tom94.net/data/publications/mueller22instant/mueller22instant-rtl.mp4">
					<span class="material-icons"> videocam </span>
					Real-Time Live
				</a>
				<a class="paper-btn" href="https://github.com/NVlabs/instant-ngp">
					<span class="material-icons"> code </span>
					Code
				</a>
			</div>
		</div>
	</div>

	<section id="teaser-videos">
		<figure style="width: 25%; float: left">
			<p class="caption_bold">
				Neural gigapixel images
			</p>
		</figure>

		<figure style="width: 25%; float: left">
			<p class="caption_bold">
				Neural SDF
			</p>
		</figure>

		<figure style="width: 25%; float: left">
			<p class="caption_bold">
				NeRF
			</p>
		</figure>

		<figure style="width: 25%; float: left">
			<p class="caption_bold">
				Neural volume
			</p>
		</figure>
		<figure style="width: 100%; float: left">
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/teaser.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>

		<figure style="width: 100%; float: left">
			<p class="caption_justify">
				We demonstrate near-instant training of neural graphics primitives on a single GPU for multiple tasks. In <b>gigapixel image</b> we represent an image by a neural network. <b>SDF</b> learns a signed distance function in 3D space whose zero level-set represents a 2D surface.
				<b>NeRF</b> <a href="https://www.matthewtancik.com/nerf">[Mildenhall et al. 2020]</a> uses 2D images and their camera poses to reconstruct a volumetric radiance-and-density field that is visualized using ray marching.
				Lastly, <b>neural volume</b> learns a denoised radiance and density field directly from a volumetric path tracer.
				In all tasks, our encoding and its efficient implementation provide clear benefits: instant training, high quality, and simplicity. Our encoding is task-agnostic: we use the same implementation and hyperparameters across all tasks and only vary the hash table size which trades off quality and performance.
				Girl With a Pearl Earring renovation &copy;Koorosh Orooj (<a href="http://profoundism.com/free_licenses.html">CC BY-SA 4.0</a>)
			</p>
		</figure>
	</section>

	<section id="news">
		<h2>News</h2>
		<hr>
		<div class="row">
			<div><span class="material-icons"> emoji_events </span> [Nov 10th 2022] Listed in <a href="https://developer.nvidia.com/blog/time-magazine-names-nvidia-instant-nerf-a-best-invention-of-2022/">TIME's Best Inventions of 2022</a>.</div>
			<div><span class="material-icons"> emoji_events </span> [July 7th 2022] Paper won the <a href="https://blog.siggraph.org/2022/07/siggraph-2022-technical-papers-awards-best-papers-and-honorable-mentions.html/">SIGGRAPH Best Paper Award</a>.</div>
			<div><span class="material-icons"> description </span> [May 3rd 2022] Paper accepted to <a href="https://s2022.siggraph.org">ACM Transactions on Graphics (SIGGRAPH 2022)</a>.</div>
			<div><span class="material-icons"> description </span> [Jan 19th 2022] Paper released on <a href="https://arxiv.org/abs/2201.05989">arXiv</a>.</div>
			<div><span class="material-icons"> integration_instructions </span> [Jan 14th 2022] Code released on <a href="https://github.com/NVlabs/instant-ngp">GitHub</a>.</div>
		</div>
	</section>

	<section id="abstract"/>
		<h2>Abstract</h2>
		<hr>
		<p>
			Neural graphics primitives, parameterized by fully connected neural networks, can be costly to train and evaluate. We reduce this cost with a versatile new input encoding that permits the use of a smaller network without sacrificing quality, thus significantly reducing the number of floating point and memory access operations. A small neural network is augmented by a multiresolution hash table of trainable feature vectors whose values are optimized through stochastic gradient descent. The multiresolution structure allows the network to disambiguate hash collisions, making for a simple architecture that is trivial to parallelize on modern GPUs. We leverage this parallelism by implementing the whole system using fully-fused CUDA kernels with a focus on minimizing wasted bandwidth and compute operations. We achieve a combined speedup of several orders of magnitude, enabling training of high-quality neural graphics primitives in a matter of seconds, and rendering in tens of milliseconds at a resolution of 1920x1080.
		</p>
	</section>

	<section id="results">
		<h2>Results</h2>
		<hr>

		<h3>Gigapixel Image</h3>
		<hr>

		<figure>
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/tokyo_online_training_counter.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>
		<p class="caption_justify">
			Real-time training progress on the image task where the neural network learns the mapping from 2D coordinates to RGB colors of a high-resolution image. Note that in this video, the network is trained from scratch—but converges so quickly you may miss it if you blink!
			Image &copy;Trevor Dobson (<a href="https://creativecommons.org/licenses/by-nc-nd/2.0/">CC BY-NC-ND 2.0</a>)<br/>
		</p>

		<h3>Neural Radiance Fields</h3>
		<hr>
		<figure style="width: 20.0%; float: left">
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/gargoyle_loop_none.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption_bold">(a) None</p>
			<p class="caption_inline">411k parameters</br>10:45 (mm:ss)</p>
		</figure>
		<!--
		<figure style="width: 20.0%; float: left">
			<p class="caption_bold">Dense grid </p>
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/gargoyle_loop_densegrid_1res.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>
		-->
		<figure style="width: 20.0%; float: left">
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/gargoyle_loop_densegrid.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption_bold">(b) Multiresolution grid</p>
			<p class="caption_inline">10k + 16.3M parameters</br>1:26 (mm:ss)</p>
		</figure>
		<figure style="width: 20.0%; float: left">
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/gargoyle_loop_frequency.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption_bold">(c) Frequency</p>
			<p class="caption_inline">438k + 0 parameters</br>13:53 (mm:ss)</p>
		</figure>
		<figure style="width: 20.0%; float: left">
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/gargoyle_loop_base_14.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption_bold">(d) Hashtable (T=2<sup>14</sup>)</p>
			<p class="caption_inline">10k + 494k parameters</br>1:40 (mm:ss)</p>
		</figure>
		<figure style="width: 20.0%; float: left">
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/gargoyle_loop_base.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption_bold">(e) Hashtable (T=2<sup>19</sup>)</p>
			<p class="caption_inline">10k + 12.6M parameters</br>1:45 (mm:ss)</p>
		</figure>
		<p class="caption_justify">
			A demonstration of the reconstruction quality of different encodings. Each configuration was trained for 11000 steps using our fast NeRF implementation, varying only the input encoding and the neural network size. The number of trainable parameters (neural network weights + encoding parameters) and training time are shown below each image. Our encoding <span class="caption_bold">(d)</span> with a similar total number of trainable parameters as the frequency encoding <span class="caption_bold">(c)</span> trains over 8 times faster, due to the sparsity of updates to the parameters and smaller neural network. Increasing the number of parameters <span class="caption_bold">(e)</span> further improves approximation quality without significantly increasing training time.
		</p>
		<figure>
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/nerf_grid_lq.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption">
				Real-time training progress on eight synthetic NeRF datsets.<br/>Drums model &copy;bryanajones (<a href="https://creativecommons.org/licenses/by/2.0/">CC BY 2.0</a>), Lego model &copy;Håvard Dalen (<a href="https://creativecommons.org/licenses/by-nc/2.0/">CC BY-NC 2.0</a>), Ship model &copy;gregzaal (<a href="https://creativecommons.org/licenses/by-sa/2.0/">CC BY-SA 2.0</a>)
			</p>
		</figure>

		<figure style="width: 49.5%; float: left">
			<video class="centered" width="100.0%" autoplay muted loop playsinline>
				<source src="assets/robot.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>

		<figure style="width: 49.5%; float: right">
			<video class="centered" width="100.0%" autoplay muted loop playsinline>
				<source src="assets/modsynth.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>
		<p class="caption_justify">
			Fly-throughs of trained real-world NeRFs. Large, natural 360 scenes (left) as well as complex scenes with many disocclusions and specular surfaces (right) are well supported.
			Both models can be rendered in real time and were trained in under 5 minutes from casually captured data: the left one from an iPhone video and the right one from 34 photographs.
		</p>

		<figure style="width: 75.0%;">
			<video class="centered" width="100.0%" autoplay muted loop playsinline>
				<source src="assets/cloud_training.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
			<p class="caption">
				We also support training NeRF-like radiance fields from the noisy output of a volumetric path tracer. Rays are fed in real-time to the network during training, which learns a denoised radiance field.<br/>
				Cloud model &copy;Walt Disney Animation Studios (<a href="https://media.disneyanimation.com/uploads/production/data_set_asset/6/asset/License_Cloud.pdf">CC BY-SA 3.0</a>)
			</p>
		</figure>

		<h3>Signed Distance Function</h3>
		<hr>
		<figure style="width: 100.0%;">
			<video class="centered" width="100.0%" autoplay muted loop playsinline>
				<source src="assets/sdf_grid_lq.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>
		<p class="caption_justify">
			Real-time training progress on various SDF datsets. Training data is generated on the fly from the ground-truth mesh using the <a href="https://developer.nvidia.com/optix">NVIDIA OptiX raytracing framework</a>.
			Bearded Man model &copy;Oliver Laric (<a href="https://creativecommons.org/licenses/by-nc-sa/3.0/">CC BY-NC-SA 3.0</a>)
		</p>

		<h3>Neural Radiance Cache</h3>
		<hr>
		<figure>
			<video class="centered" width="100%" autoplay muted loop playsinline>
				<source src="assets/nrc_new_vs_old.mp4" type="video/mp4">
				Your browser does not support the video tag.
			</video>
		</figure>
		<p class="caption_justify">
			Direct visualization of a <em>neural radiance cache</em>, in which the network predicts outgoing radiance at the first non-specular vertex of each pixel's path, and is trained on-line from rays generated by a real-time pathtracer. On the left, we show results using the triangle wave encoding of <a href="https://research.nvidia.com/publication/2021-06_Real-time-Neural-Radiance">[Müller et al. 2021]</a>; on the right, the new multiresolution hash encoding allows the network to learn much sharper details, for example in the shadow regions.
		</p>

	</section>

	<section id="paper">
		<h2>Paper</h2>
		<hr>
		<div class="flex-row">
			<div style="box-sizing: border-box; padding: 16px; margin: auto;">
				<a href="assets/mueller2022instant.pdf"><img class="screenshot" src="assets/paper-thumbnail.png"></a>
			</div>
			<div style="width: 60%">
				<p><b>Instant Neural Graphics Primitives with a Multiresolution Hash Encoding</b></p>
				<p>Thomas Müller, Alex Evans, Christoph Schied, Alexander Keller</p>

				<div><span class="material-icons"> description </span><a href="assets/mueller2022instant.pdf"> Paper preprint (PDF, 17.7 MB)</a></div>
				<div><span class="material-icons"> description </span><a href="https://arxiv.org/abs/2201.05989"> arXiv version</a></div>
				<div><span class="material-icons"> insert_comment </span><a href="assets/mueller2022instant.bib"> BibTeX</a></div>
				<div><span class="material-icons"> integration_instructions </span><a href="https://github.com/NVlabs/instant-ngp"> Code</a></div>
				<div><span class="material-icons"> videocam </span><a href="assets/mueller2022instant.mp4"> Video</a></div>

				<p>Please send feedback and questions to <a href="https://tom94.net">Thomas Müller</a></p>
			</div>
		</div>
	</section>

	<section id="bibtex">
		<h2>Citation</h2>
		<hr>
		<pre><code>@article{mueller2022instant,
    author = {Thomas M\"uller and Alex Evans and Christoph Schied and Alexander Keller},
    title = {Instant Neural Graphics Primitives with a Multiresolution Hash Encoding},
    journal = {ACM Trans. Graph.},
    issue_date = {July 2022},
    volume = {41},
    number = {4},
    month = jul,
    year = {2022},
    pages = {102:1--102:15},
    articleno = {102},
    numpages = {15},
    url = {https://doi.org/10.1145/3528223.3530127},
    doi = {10.1145/3528223.3530127},
    publisher = {ACM},
    address = {New York, NY, USA}
}
</code></pre>
	</section>

	<section id="acknowledgements">
		<h2>Acknowledgements</h2>
		<hr>
		<div class="row">
			<p>
			We would like to thank
			<a href="https://anjulpatney.com/">Anjul Patney</a>,
			<a href="https://luebke.us/">David Luebke</a>,
			<a href="https://research.nvidia.com/person/jacob-munkberg">Jacob Munkberg</a>,
			<a href="http://granskog.xyz/">Jonathan Granskog</a>,
			<a href="https://www.cs.mcgill.ca/~jtremb59/">Jonathan Tremblay</a>,
			<a href="https://luminohope.org/">Koki Nagano</a>,
			<a href="https://research.nvidia.com/person/marco-salvi">Marco Salvi</a>,
			<a href="https://research.nvidia.com/person/nikolaus-binder">Nikolaus Binder</a>,
			<a href="https://www.cs.toronto.edu/~jlucas/">James Lucas</a>, and
			<a href="https://tovacinni.github.io">Towaki Takikawa</a>
			for proof-reading, feedback, profound discussions, and early testing.
			We also thank <a href="https://joeylitalien.github.io/">Joey Litalien</a> for providing us with the framework for this website.
			<br/>
			<br/>
			<em>Girl With a Pearl Earring</em> renovation &copy;Koorosh Orooj (<a href="http://profoundism.com/free_licenses.html">CC BY-SA 4.0</a>)
			<br/>
			<em>Tokyo</em> gigapixel image &copy;Trevor Dobson (<a href="https://creativecommons.org/licenses/by-nc-nd/2.0/">CC BY-NC-ND 2.0</a>)
			<br/>
			<em>Detailed Drum Set</em> &copy;bryanajones (<a href="https://creativecommons.org/licenses/by/2.0/">CC BY 2.0</a>)
			<br/>
			<em>Lego 856 Bulldozer</em> &copy;Håvard Dalen (<a href="https://creativecommons.org/licenses/by-nc/2.0/">CC BY-NC 2.0</a>)
			<br/>
			<em>Suzanne's Revenge</em> ship &copy;gregzaal (<a href="https://creativecommons.org/licenses/by-sa/2.0/">CC BY-SA 2.0</a>)
			<br/>
			<em>Lucy</em> model from the <a href="http://graphics.stanford.edu/data/3Dscanrep/">Stanford 3D scan repository</a>
			<br/>
			<em>Factory robot</em> dataset by Arman Toorians and Saurabh Jain
			<br/>
			<em>Disney Cloud</em> model &copy;Walt Disney Animation Studios (<a href="https://media.disneyanimation.com/uploads/production/data_set_asset/6/asset/License_Cloud.pdf">CC BY-SA 3.0</a>)
			<br/>
			<em>Bearded Man</em> model &copy;Oliver Laric (<a href="https://creativecommons.org/licenses/by-nc-sa/3.0/">CC BY-NC-SA 3.0</a>)
			</p>
		</div>
	</section>
</div>
</body>
</html>