import{s as Ga,n as Ha,o as $a}from"../chunks/scheduler.7bc62968.js";import{S as Xa,i as Sa,g as i,s as n,r as E,m as Ys,H as ta,A as Qa,h as p,f as e,c as l,j as ns,u as Y,x as m,n as Ps,B as na,k as Fa,y as P,a as t,v as D,d as K,t as O,w as ss}from"../chunks/index.2f8492b0.js";import{C as Aa}from"../chunks/CodeBlock.4e96d6f7.js";import{H as ls}from"../chunks/Heading.14fb2476.js";function La(la){let r,is,as,ps,c,ms,h,ia=`Neural Radiance Fields are a way of storing a 3D scene within a neural network. This way of storing and representing a scene is often called an implicit representation, since the scene parameters are fully represented by the underlying Multi-Layer Perceptron (MLP).
(As compared to an explicit representation that stores scene parameters like colour or density explicitly in voxel grids.)
This novel way of representing a scene showed very impressive results in the task of <a href="https://en.wikipedia.org/wiki/View_synthesis" rel="nofollow">novel view synthesis</a>, the task of interpolating novel views from camera perspectives that are not in the training set.
Furthermore, it allows us to store large scenes with a smaller memory footprint than explicit representation, since we merely need to store the weights of our neural network compared to voxel grids, which increase in memory size by a cubic term.`,rs,u,os,y,pa=`The field of NeRFs is relatively young with the first publication by <a href="https://www.matthewtancik.com/nerf" rel="nofollow">Mildenhall et al.</a> appearing in 2020.
Since then, a vast number of papers have been published and fast advancements have been made.
Since 2020, more than 620 preprints and publications have been released, with more than 250 repositories on GitHub. <em>(as of Dec 2023, statistics from <a href="https://paperswithcode.com/method/nerf" rel="nofollow">paperswithcode.com</a>)</em>`,cs,d,ma=`Since the first formulation of NeRFs requires long training times (up to days on beefy GPUs), there have been a lot of advancements towards faster training and inference.
An important leap was NVIDIA’s <a href="https://nvlabs.github.io/instant-ngp/" rel="nofollow">Instant-ngp</a>, which was released in 2022.
While the model architecture used in this approach is similar to existing ones, the authors introduced a novel encoding method that uses trainable hash-tables.
Thanks to this type of encoding, we can shrink the MLP down significantly without loosing reconstruction quality.
This novel approach was faster to train and query while performing on par quality wise with then state-of-the-art methods.
<a href="https://jonbarron.info/mipnerf360/" rel="nofollow">Mipnerf-360</a>, which was also released in 2022, is also worth mentioning.
Again, the model architecture is the same as for most NeRFs, but the authors introduced a novel scene contraction that allows us to represent scenes that are unbounded in all directions, which is important for real-world applications.
<a href="https://jonbarron.info/zipnerf/" rel="nofollow">Zip-NeRF</a>, released in 2023, combines recent advancements like the encoding from <a href="https://nvlabs.github.io/instant-ngp/" rel="nofollow">Instant-ngp</a> and the scene contraction from <a href="https://jonbarron.info/mipnerf360/" rel="nofollow">Mipnerf-360</a> to handle real-world situation whilst decreasing training times to under an hour.
<em>(this is still measured on beefy GPUs to be fair)</em>`,hs,g,ra="Since the field of NeRFs is rapidly evolving, we added a section <code>sota</code> at the end where we will tease the latest research and the possible future direction of NeRFs.",us,f,oa="But now enough with the history, let’s dive into the intrinsics of NeRFs! 🚀🚀",ys,v,ds,b,ca=`The fundamental idea behind NeRFs is to represent a scene as a continuous function that maps a position, $\\mathbf{x} \\in \\mathbb{R}^{3}$, and a viewing direction, $\\boldsymbol{\\theta} \\in \\mathbb{R}^{2}$, to a colour $\\mathbf{c} \\in \\mathbb{R}^{3}$ and volume density $\\sigma \\in \\mathbb{R}^{1}$.
As neural networks can serve as universal function approximators, we can approximate this continuous function that represents the scene with a simple Multi-Layer Perceptron (MLP) $F_{\\mathrm{\\Theta}} : (\\mathbf{x}, \\boldsymbol{\\theta}) \\to (\\mathbf{c},\\sigma)$.`,gs,w,ha="A simple NeRF pipeline can be summarized with the following picture:",fs,M,ua='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/nerf_pipeline.png" alt="nerf_pipeline"/>',vs,T,ya='Image from: <a href="https://www.matthewtancik.com/nerf" rel="nofollow">Mildenhall et al. (2020)</a>',bs,J,da="<strong>(a)</strong> Sample points and viewing directions along camera rays and pass them through the network.",ws,U,ga="<strong>(b)</strong> Network output is a colour vector and density value for each sample.",Ms,j,fa="<strong>(c)</strong> Combine the outputs of the network via volumetric rendering to go from discrete samples in 3D space to a 2D image.",Ts,C,va="<strong>(d)</strong> Compute the loss and update network gradients via backpropagation to represent scene.",Js,x,ba="This is overview is very high level, so for a better understanding, let’s go into the details of the volume rendering and the used loss function.",Us,_,wa="<strong>Volume Rendering</strong>",js,o,Ds,q,Ma="differentiable",Ks,Cs,Na='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi mathvariant="bold">C</mi><mo stretchy="false">(</mo><mi mathvariant="bold">r</mi><mo stretchy="false">)</mo><mo>=</mo><msubsup><mo>∫</mo><msub><mi>t</mi><mi>n</mi></msub><msub><mi>t</mi><mi>f</mi></msub></msubsup><mi>T</mi><mo stretchy="false">(</mo><mi>t</mi><mo stretchy="false">)</mo><mi>σ</mi><mo stretchy="false">(</mo><mi mathvariant="bold">r</mi><mo stretchy="false">(</mo><mi>t</mi><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mi mathvariant="bold">c</mi><mo stretchy="false">(</mo><mi mathvariant="bold">r</mi><mo stretchy="false">(</mo><mi>t</mi><mo stretchy="false">)</mo><mo separator="true">,</mo><mi mathvariant="bold">d</mi><mo stretchy="false">)</mo><mi>d</mi><mi>t</mi></mrow><annotation encoding="application/x-tex">\\mathbf{C}(\\mathbf{r}) = \\int_{t_n}^{t_f}T(t)\\sigma(\\mathbf{r}(t))\\mathbf{c}(\\mathbf{r}(t),\\mathbf{d})dt</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathbf">C</span><span class="mopen">(</span><span class="mord mathbf">r</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.5555em;vertical-align:-1.012em;"></span><span class="mop"><span class="mop op-symbol large-op" style="margin-right:0.44445em;position:relative;top:-0.0011em;">∫</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.5435em;"><span style="top:-1.7881em;margin-left:-0.4445em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight">t</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1645em;"><span style="top:-2.357em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight">n</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.143em;"><span></span></span></span></span></span></span></span></span></span><span style="top:-3.8129em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathnormal mtight">t</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3448em;"><span style="top:-2.3488em;margin-left:0em;margin-right:0.0714em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mathnormal mtight" style="margin-right:0.10764em;">f</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2901em;"><span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.012em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="mopen">(</span><span class="mord mathnormal">t</span><span class="mclose">)</span><span class="mord mathnormal" style="margin-right:0.03588em;">σ</span><span class="mopen">(</span><span class="mord mathbf">r</span><span class="mopen">(</span><span class="mord mathnormal">t</span><span class="mclose">))</span><span class="mord mathbf">c</span><span class="mopen">(</span><span class="mord mathbf">r</span><span class="mopen">(</span><span class="mord mathnormal">t</span><span class="mclose">)</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathbf">d</span><span class="mclose">)</span><span class="mord mathnormal">d</span><span class="mord mathnormal">t</span></span></span></span></span>',xs,I,Ta=`In the equation above, $\\mathbf{C}(\\mathbf{r})$ is the expected colour of a camera ray $\\mathbf{r}(t)=\\mathbf{o}+t\\mathbf{d}$, where $\\mathbf{o} \\in \\mathbb{R}^{3}$ is the origin of the camera, $\\boldsymbol{d} \\in \\mathbb{R}^{3}$ is the viewing direction as a 3D unit vector and $t \\in \\mathbb{R}_+$ is the distance along the ray.
$t_n$ and $t_f$ stand for the near and far bounds of the ray, respectively.
$T(t)$ denotes the accumulated transmittance along ray $\\mathbf{r}(t)$ from $t_n$ to $t$.`,_s,z,Os,Is,Ba='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi><mover accent="true"><mi mathvariant="bold-italic">C</mi><mo>^</mo></mover></mi><mo stretchy="false">(</mo><mi mathvariant="bold">r</mi><mo stretchy="false">)</mo><mo>=</mo><munderover><mo>∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow><mi>N</mi></munderover><msub><mi>T</mi><mi>i</mi></msub><mo stretchy="false">(</mo><mn>1</mn><mo>−</mo><mi>exp</mi><mo>⁡</mo><mo stretchy="false">(</mo><mo>−</mo><msub><mi>σ</mi><mi>i</mi></msub><msub><mi>δ</mi><mi>i</mi></msub><mo stretchy="false">)</mo><mo stretchy="false">)</mo><msub><mi mathvariant="bold">c</mi><mi>i</mi></msub><mtext> </mtext><mo separator="true">,</mo><mtext> where </mtext><msub><mi>T</mi><mi>i</mi></msub><mo>=</mo><mi>exp</mi><mo>⁡</mo><mo fence="false" stretchy="true" minsize="2.4em" maxsize="2.4em">(</mo><mo>−</mo><munderover><mo>∑</mo><mrow><mi>j</mi><mo>=</mo><mn>1</mn></mrow><mrow><mi>i</mi><mo>−</mo><mn>1</mn></mrow></munderover><msub><mi>σ</mi><mi>j</mi></msub><msub><mi>δ</mi><mi>j</mi></msub><mo fence="false" stretchy="true" minsize="2.4em" maxsize="2.4em">)</mo></mrow><annotation encoding="application/x-tex">\\boldsymbol{\\hat{C}}(\\mathbf{r})=\\sum_{i=1}^{N}T_i (1-\\exp(-\\sigma_i \\delta_i)) \\mathbf{c}_i\\,, \\textrm{ where }T_i=\\exp \\bigg(-\\sum_{j=1}^{i-1} \\sigma_j \\delta_j \\bigg)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.1995em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord"><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.9495em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord boldsymbol" style="margin-right:0.06979em;">C</span></span><span style="top:-3.2551em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.2875em;"><span class="mord mathbf">^</span></span></span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord mathbf">r</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:3.106em;vertical-align:-1.2777em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.8283em;"><span style="top:-1.8723em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mrel mtight">=</span><span class="mord mtight">1</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span><span style="top:-4.3em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.10903em;">N</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.2777em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord">1</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mop">exp</span><span class="mopen">(</span><span class="mord">−</span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">σ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03785em;">δ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0379em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mclose">))</span><span class="mord"><span class="mord mathbf">c</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord text"><span class="mord textrm"> where </span></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.4em;vertical-align:-0.95em;"></span><span class="mop">exp</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="delimsizing size3">(</span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:3.2254em;vertical-align:-1.4138em;"></span><span class="mop op-limits"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.8117em;"><span style="top:-1.8723em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.05724em;">j</span><span class="mrel mtight">=</span><span class="mord mtight">1</span></span></span></span><span style="top:-3.05em;"><span class="pstrut" style="height:3.05em;"></span><span><span class="mop op-symbol large-op">∑</span></span></span><span style="top:-4.3em;margin-left:0em;"><span class="pstrut" style="height:3.05em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">i</span><span class="mbin mtight">−</span><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:1.4138em;"><span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03588em;">σ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05724em;">j</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.03785em;">δ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3117em;"><span style="top:-2.55em;margin-left:-0.0379em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05724em;">j</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span><span class="mord"><span class="delimsizing size3">)</span></span></span></span></span></span>',zs,k,Ja="Below, you can see a schematic visualisation of a discretized camera ray in order to get a better sense of the variables from above:",ks,R,Ua='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/nerf_ray_visualisation.png" alt="ray_image"/>',Rs,Z,ja="<strong>Loss formulation</strong>",Zs,W,sa,Ws,Va=`<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><msub><mi mathvariant="script">L</mi><mrow><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi></mrow></msub><mo stretchy="false">(</mo><mi><mover accent="true"><mi mathvariant="bold-italic">C</mi><mo>^</mo></mover></mi><mo separator="true">,</mo><mi><msup><mi mathvariant="bold-italic">C</mi><mo mathvariant="bold-italic">∗</mo></msup></mi><mo stretchy="false">)</mo><mo>=</mo><msup><mrow><mo fence="true">∥</mo><mi><mover accent="true"><mi mathvariant="bold-italic">C</mi><mo>^</mo></mover></mi><mo>−</mo><mi><msup><mi mathvariant="bold-italic">C</mi><mo mathvariant="bold-italic">∗</mo></msup></mi><mo fence="true">∥</mo></mrow><mn>2</mn></msup></mrow><annotation encoding="application/x-tex">\\mathcal{L}_{\\rm recon}(\\boldsymbol{\\hat{C}},\\boldsymbol{C^*}) = \\left\\|\\boldsymbol{\\hat{C}}-\\boldsymbol{C^*}\\right\\|^2</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1.1995em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathcal">L</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathrm mtight">recon</span></span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mopen">(</span><span class="mord"><span class="mord"><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.9495em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord boldsymbol" style="margin-right:0.06979em;">C</span></span><span style="top:-3.2551em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.2875em;"><span class="mord mathbf">^</span></span></span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord"><span class="mord"><span class="mord"><span class="mord boldsymbol" style="margin-right:0.06979em;">C</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.7436em;"><span style="top:-3.113em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mbin mathbf mtight">∗</span></span></span></span></span></span></span></span></span></span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.004em;vertical-align:-0.65em;"></span><span class="minner"><span class="minner"><span class="mopen"><span class="delimsizing mult"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.15em;"><span style="top:-3.15em;"><span class="pstrut" style="height:3.8em;"></span><span style="width:0.556em;height:1.800em;"><svg xmlns="http://www.w3.org/2000/svg" width='0.556em' height='1.800em' viewBox='0 0 556 1800'><path d='M145 15 v585 v600 v585 c2.667,10,9.667,15,21,15
c10,0,16.667,-5,20,-15 v-585 v-600 v-585 c-2.667,-10,-9.667,-15,-21,-15
c-10,0,-16.667,5,-20,15z M188 15 H145 v585 v600 v585 h43z
M367 15 v585 v600 v585 c2.667,10,9.667,15,21,15
c10,0,16.667,-5,20,-15 v-585 v-600 v-585 c-2.667,-10,-9.667,-15,-21,-15
c-10,0,-16.667,5,-20,15z M410 15 H367 v585 v600 v585 h43z'/></svg></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.65em;"><span></span></span></span></span></span></span><span class="mord"><span class="mord"><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.9495em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord boldsymbol" style="margin-right:0.06979em;">C</span></span><span style="top:-3.2551em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.2875em;"><span class="mord mathbf">^</span></span></span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mord"><span class="mord"><span class="mord"><span class="mord boldsymbol" style="margin-right:0.06979em;">C</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.7436em;"><span style="top:-3.113em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mbin mathbf mtight">∗</span></span></span></span></span></span></span></span></span></span><span class="mclose"><span class="delimsizing mult"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.15em;"><span style="top:-3.15em;"><span class="pstrut" style="height:3.8em;"></span><span style="width:0.556em;height:1.800em;"><svg xmlns="http://www.w3.org/2000/svg" width='0.556em' height='1.800em' viewBox='0 0 556 1800'><path d='M145 15 v585 v600 v585 c2.667,10,9.667,15,21,15
c10,0,16.667,-5,20,-15 v-585 v-600 v-585 c-2.667,-10,-9.667,-15,-21,-15
c-10,0,-16.667,5,-20,15z M188 15 H145 v585 v600 v585 h43z
M367 15 v585 v600 v585 c2.667,10,9.667,15,21,15
c10,0,16.667,-5,20,-15 v-585 v-600 v-585 c-2.667,-10,-9.667,-15,-21,-15
c-10,0,-16.667,5,-20,15z M410 15 H367 v585 v600 v585 h43z'/></svg></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.65em;"><span></span></span></span></span></span></span></span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:1.354em;"><span style="top:-3.6029em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span></span></span></span></span></span></span></span></span>`,Fs,F,Ca=",where $\\boldsymbol{\\hat{C}}$ is the rendered pixel colour and $\\boldsymbol{C}^*$ is the ground truth pixel colour.",Ns,N,xa="<strong>Additional remarks</strong>",Bs,B,_a=`It is very hard to describe the whole NeRF pipeline in detail within a single chapter.
The explanations above are important to understand the basic concepts and similar if not identical in every NeRF model.
However, some additional tricks are needed to obtain a well performing model.`,Vs,V,Ia=`First of all, it is necesarry to encode input signals in order to capture high-frequency variations in colour and geometry.
The practice of encoding inputs before passing them through a neural network is not unique to the NeRF domain but also widely adopted in other ML domains like for example Natural Language Processing (NLP).
A very simple encoding where we map the inputs to a higher dimensional space, enabling us to capture high frequency variations in scene parameters could look as follows:`,Gs,G,Hs,H,za="The output should look something like the image below:",$s,$,ka='<img src="https://huggingface.co/datasets/hf-vision/course-assets/blob/main/nerf_encodings.png" alt="encoding"/>',Xs,X,Ra=`The second trick worth mentioning is that most methods use smart approaches to sample points in space.
Essentially, we want to avoid sampling in regions where the scene is empty.
There are various approaches to concentrate samples in regions that contribute most to the final image, but the most prominent one is to use a second network, often called <em>proposal network</em> so that no compute is wasted.
If you are interested in the inner workings and optimisation of such a <em>proposal network</em>, feel free to dig into the publication of <a href="https://jonbarron.info/mipnerf360/" rel="nofollow">Mipnerf-360</a>, where it was first proposed.`,Ss,S,Qs,Q,Za=`To get the full experience when training your first NeRF, I recommend taking a look at the awesome <a href="https://colab.research.google.com/github/nerfstudio-project/nerfstudio/blob/main/colab/demo.ipynb" rel="nofollow">Google Colab notebook from the nerfstudio team</a>.
There, you can upload images of a scene of your choice and train a NeRF. You could for example fit a model to represent your living room. 🎉🎉`,As,A,Ls,L,Wa=`The field is rapidly evolving and the number of new publications is almost exploding.
Concerning training and rendering speed, <a href="https://vr-nerf.github.io" rel="nofollow">VR-NeRF</a> and <a href="https://smerf-3d.github.io" rel="nofollow">SMERF</a> show very promising results.
We believe that we will soon be able to stream a real-world scene in real-time on an edge device, and this is a huge leap towards a realistic <em>Metaverse</em>.
However, the research in the field of NeRFs is not only focusing on training and inference speed, but encompasses various directions like, Generative NeRFs, Pose Estimation, Deformable NeRFs, Compositionality and many more.
If you are interested in a curated list of NeRF publications, checkout <a href="https://github.com/awesome-NeRF/awesome-NeRF" rel="nofollow">Awesome-NeRF</a>.`,qs,es,Es;return c=new ls({props:{title:"Neural Radiance Fields (NeRFs)",local:"neural-radiance-fields-nerfs",headingTag:"h1"}}),u=new ls({props:{title:"Short History 📖",local:"short-history-",headingTag:"h2"}}),v=new ls({props:{title:"Underlying approach (Vanilla NeRF) 📘🔍",local:"underlying-approach-vanilla-nerf-",headingTag:"h2"}}),G=new Aa({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBtZWRpYXB5JTIwYXMlMjBtZWRpYSUwQWltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUwQSUwQSUwQWRlZiUyMHBvc2l0aW9uYWxfZW5jb2RpbmcoaW5fdGVuc29yJTJDJTIwbnVtX2ZyZXF1ZW5jaWVzJTJDJTIwbWluX2ZyZXFfZXhwJTJDJTIwbWF4X2ZyZXFfZXhwKSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkZ1bmN0aW9uJTIwZm9yJTIwcG9zaXRpb25hbCUyMGVuY29kaW5nLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMyUyMFNjYWxlJTIwaW5wdXQlMjB0ZW5zb3IlMjB0byUyMCU1QjAlMkMlMjAyJTIwKiUyMHBpJTVEJTBBJTIwJTIwJTIwJTIwc2NhbGVkX2luX3RlbnNvciUyMCUzRCUyMDIlMjAqJTIwbnAucGklMjAqJTIwaW5fdGVuc29yJTBBJTIwJTIwJTIwJTIwJTIzJTIwR2VuZXJhdGUlMjBmcmVxdWVuY3klMjBzcGVjdHJ1bSUwQSUyMCUyMCUyMCUyMGZyZXFzJTIwJTNEJTIwMiUyMCoqJTIwdG9yY2gubGluc3BhY2UoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWluX2ZyZXFfZXhwJTJDJTIwbWF4X2ZyZXFfZXhwJTJDJTIwbnVtX2ZyZXF1ZW5jaWVzJTJDJTIwZGV2aWNlJTNEaW5fdGVuc29yLmRldmljZSUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjMlMjBHZW5lcmF0ZSUyMGVuY29kaW5ncyUwQSUyMCUyMCUyMCUyMHNjYWxlZF9pbnB1dHMlMjAlM0QlMjBzY2FsZWRfaW5fdGVuc29yLnVuc3F1ZWV6ZSgtMSklMjAqJTIwZnJlcXMlMEElMjAlMjAlMjAlMjBlbmNvZGVkX2lucHV0cyUyMCUzRCUyMHRvcmNoLmNhdCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUJ0b3JjaC5zaW4oc2NhbGVkX2lucHV0cyklMkMlMjB0b3JjaC5jb3Moc2NhbGVkX2lucHV0cyklNUQlMkMlMjBkaW0lM0QtMSUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBlbmNvZGVkX2lucHV0cy52aWV3KCppbl90ZW5zb3Iuc2hhcGUlNUIlM0EtMSU1RCUyQyUyMC0xKSUwQSUwQSUwQWRlZiUyMHZpc3VhbGl6ZV9ncmlkKGdyaWQlMkMlMjBlbmNvZGVkX2ltYWdlcyUyQyUyMHJlc29sdXRpb24pJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIySGVscGVyJTIwRnVuY3Rpb24lMjB0byUyMHZpc3VhbGl6ZSUyMGdyaWQuJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwU3BsaXQlMjB0aGUlMjBncmlkJTIwaW50byUyMHNlcGFyYXRlJTIwY2hhbm5lbHMlMjBmb3IlMjB4JTIwYW5kJTIweSUwQSUyMCUyMCUyMCUyMHhfY2hhbm5lbCUyQyUyMHlfY2hhbm5lbCUyMCUzRCUyMGdyaWQlNUIuLi4lMkMlMjAwJTVEJTJDJTIwZ3JpZCU1Qi4uLiUyQyUyMDElNUQlMEElMjAlMjAlMjAlMjAlMjMlMjBTaG93JTIwdGhlJTIwb3JpZ2luYWwlMjBncmlkJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIySW5wdXQlMjBWYWx1ZXMlM0ElMjIpJTBBJTIwJTIwJTIwJTIwbWVkaWEuc2hvd19pbWFnZXMoJTVCeF9jaGFubmVsJTJDJTIweV9jaGFubmVsJTVEJTJDJTIwY21hcCUzRCUyMnBsYXNtYSUyMiUyQyUyMGJvcmRlciUzRFRydWUpJTBBJTIwJTIwJTIwJTIwJTIzJTIwU2hvdyUyMHRoZSUyMGVuY29kZWQlMjBncmlkJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIyRW5jb2RlZCUyMFZhbHVlcyUzQSUyMiklMEElMjAlMjAlMjAlMjBudW1fY2hhbm5lbHNfdG9fdmlzdWFsaXplJTIwJTNEJTIwbWluKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDglMkMlMjBlbmNvZGVkX2ltYWdlcy5zaGFwZSU1Qi0xJTVEJTBBJTIwJTIwJTIwJTIwKSUyMCUyMCUyMyUyMFZpc3VhbGl6ZSUyMHVwJTIwdG8lMjA4JTIwY2hhbm5lbHMlMEElMjAlMjAlMjAlMjBlbmNvZGVkX2ltYWdlc190b19zaG93JTIwJTNEJTIwZW5jb2RlZF9pbWFnZXMudmlldyhyZXNvbHV0aW9uJTJDJTIwcmVzb2x1dGlvbiUyQyUyMC0xKS5wZXJtdXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDIlMkMlMjAwJTJDJTIwMSUwQSUyMCUyMCUyMCUyMCklNUIlM0FudW1fY2hhbm5lbHNfdG9fdmlzdWFsaXplJTVEJTBBJTIwJTIwJTIwJTIwbWVkaWEuc2hvd19pbWFnZXMoZW5jb2RlZF9pbWFnZXNfdG9fc2hvdyUyQyUyMHZtaW4lM0QtMSUyQyUyMHZtYXglM0QxJTJDJTIwY21hcCUzRCUyMnBsYXNtYSUyMiUyQyUyMGJvcmRlciUzRFRydWUpJTBBJTBBJTBBJTIzJTIwUGFyYW1ldGVycyUyMHNpbWlsYXIlMjB0byUyMHlvdXIlMjBOZVJGRW5jb2RpbmclMjBleGFtcGxlJTBBbnVtX2ZyZXF1ZW5jaWVzJTIwJTNEJTIwNCUwQW1pbl9mcmVxX2V4cCUyMCUzRCUyMDAlMEFtYXhfZnJlcV9leHAlMjAlM0QlMjA2JTBBcmVzb2x1dGlvbiUyMCUzRCUyMDEyOCUwQSUwQSUyMyUyMEdlbmVyYXRlJTIwYSUyMDJEJTIwZ3JpZCUyMG9mJTIwcG9pbnRzJTIwaW4lMjB0aGUlMjByYW5nZSUyMCU1QjAlMkMlMjAxJTVEJTBBeF9zYW1wbGVzJTIwJTNEJTIwdG9yY2gubGluc3BhY2UoMCUyQyUyMDElMkMlMjByZXNvbHV0aW9uKSUwQXlfc2FtcGxlcyUyMCUzRCUyMHRvcmNoLmxpbnNwYWNlKDAlMkMlMjAxJTJDJTIwcmVzb2x1dGlvbiklMEFncmlkJTIwJTNEJTIwdG9yY2guc3RhY2soJTBBJTIwJTIwJTIwJTIwdG9yY2gubWVzaGdyaWQoeF9zYW1wbGVzJTJDJTIweV9zYW1wbGVzKSUyQyUyMGRpbSUzRC0xJTBBKSUyMCUyMCUyMyUyMCU1QnJlc29sdXRpb24lMkMlMjByZXNvbHV0aW9uJTJDJTIwMiU1RCUwQSUwQSUyMyUyMEFwcGx5JTIwcG9zaXRpb25hbCUyMGVuY29kaW5nJTBBZW5jb2RlZF9ncmlkJTIwJTNEJTIwcG9zaXRpb25hbF9lbmNvZGluZyhncmlkJTJDJTIwbnVtX2ZyZXF1ZW5jaWVzJTJDJTIwbWluX2ZyZXFfZXhwJTJDJTIwbWF4X2ZyZXFfZXhwKSUwQSUwQSUyMyUyMFZpc3VhbGl6ZSUyMHJlc3VsdCUwQXZpc3VhbGl6ZV9ncmlkKGdyaWQlMkMlMjBlbmNvZGVkX2dyaWQlMkMlMjByZXNvbHV0aW9uKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> mediapy <span class="hljs-keyword">as</span> media
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">positional_encoding</span>(<span class="hljs-params">in_tensor, num_frequencies, min_freq_exp, max_freq_exp</span>):
    <span class="hljs-string">&quot;&quot;&quot;Function for positional encoding.&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Scale input tensor to [0, 2 * pi]</span>
    scaled_in_tensor = <span class="hljs-number">2</span> * np.pi * in_tensor
    <span class="hljs-comment"># Generate frequency spectrum</span>
    freqs = <span class="hljs-number">2</span> ** torch.linspace(
        min_freq_exp, max_freq_exp, num_frequencies, device=in_tensor.device
    )
    <span class="hljs-comment"># Generate encodings</span>
    scaled_inputs = scaled_in_tensor.unsqueeze(-<span class="hljs-number">1</span>) * freqs
    encoded_inputs = torch.cat(
        [torch.sin(scaled_inputs), torch.cos(scaled_inputs)], dim=-<span class="hljs-number">1</span>
    )
    <span class="hljs-keyword">return</span> encoded_inputs.view(*in_tensor.shape[:-<span class="hljs-number">1</span>], -<span class="hljs-number">1</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">visualize_grid</span>(<span class="hljs-params">grid, encoded_images, resolution</span>):
    <span class="hljs-string">&quot;&quot;&quot;Helper Function to visualize grid.&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Split the grid into separate channels for x and y</span>
    x_channel, y_channel = grid[..., <span class="hljs-number">0</span>], grid[..., <span class="hljs-number">1</span>]
    <span class="hljs-comment"># Show the original grid</span>
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Input Values:&quot;</span>)
    media.show_images([x_channel, y_channel], cmap=<span class="hljs-string">&quot;plasma&quot;</span>, border=<span class="hljs-literal">True</span>)
    <span class="hljs-comment"># Show the encoded grid</span>
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Encoded Values:&quot;</span>)
    num_channels_to_visualize = <span class="hljs-built_in">min</span>(
        <span class="hljs-number">8</span>, encoded_images.shape[-<span class="hljs-number">1</span>]
    )  <span class="hljs-comment"># Visualize up to 8 channels</span>
    encoded_images_to_show = encoded_images.view(resolution, resolution, -<span class="hljs-number">1</span>).permute(
        <span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>
    )[:num_channels_to_visualize]
    media.show_images(encoded_images_to_show, vmin=-<span class="hljs-number">1</span>, vmax=<span class="hljs-number">1</span>, cmap=<span class="hljs-string">&quot;plasma&quot;</span>, border=<span class="hljs-literal">True</span>)


<span class="hljs-comment"># Parameters similar to your NeRFEncoding example</span>
num_frequencies = <span class="hljs-number">4</span>
min_freq_exp = <span class="hljs-number">0</span>
max_freq_exp = <span class="hljs-number">6</span>
resolution = <span class="hljs-number">128</span>

<span class="hljs-comment"># Generate a 2D grid of points in the range [0, 1]</span>
x_samples = torch.linspace(<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, resolution)
y_samples = torch.linspace(<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, resolution)
grid = torch.stack(
    torch.meshgrid(x_samples, y_samples), dim=-<span class="hljs-number">1</span>
)  <span class="hljs-comment"># [resolution, resolution, 2]</span>

<span class="hljs-comment"># Apply positional encoding</span>
encoded_grid = positional_encoding(grid, num_frequencies, min_freq_exp, max_freq_exp)

<span class="hljs-comment"># Visualize result</span>
visualize_grid(grid, encoded_grid, resolution)`,wrap:!1}}),S=new ls({props:{title:"Train your own NeRF",local:"train-your-own-nerf",headingTag:"h2"}}),A=new ls({props:{title:"Current advancements in the field",local:"sota",headingTag:"h2"}}),{c(){r=i("meta"),is=n(),as=i("p"),ps=n(),E(c.$$.fragment),ms=n(),h=i("p"),h.innerHTML=ia,rs=n(),E(u.$$.fragment),os=n(),y=i("p"),y.innerHTML=pa,cs=n(),d=i("p"),d.innerHTML=ma,hs=n(),g=i("p"),g.innerHTML=ra,us=n(),f=i("p"),f.textContent=oa,ys=n(),E(v.$$.fragment),ds=n(),b=i("p"),b.textContent=ca,gs=n(),w=i("p"),w.textContent=ha,fs=n(),M=i("p"),M.innerHTML=ua,vs=n(),T=i("p"),T.innerHTML=ya,bs=n(),J=i("p"),J.innerHTML=da,ws=n(),U=i("p"),U.innerHTML=ga,Ms=n(),j=i("p"),j.innerHTML=fa,Ts=n(),C=i("p"),C.innerHTML=va,Js=n(),x=i("p"),x.textContent=ba,Us=n(),_=i("p"),_.innerHTML=wa,js=n(),o=i("p"),Ds=Ys(`The principles behind the process of volumetric rendering are well established in classical computer graphics pipelines and do not stem from NeRFs.
What is important for the use case of NeRFs is that this step is `),q=i("strong"),q.textContent=Ma,Ks=Ys(` in order to allow for backpropagation. The simplest form of volumetric rendering in NeRFs can be formulated as follows:
`),Cs=new ta(!1),xs=n(),I=i("p"),I.textContent=Ta,_s=n(),z=i("p"),Os=Ys(`After discretization, the equation above can be computed as the following sum:
`),Is=new ta(!1),zs=n(),k=i("p"),k.textContent=Ja,ks=n(),R=i("p"),R.innerHTML=Ua,Rs=n(),Z=i("p"),Z.innerHTML=ja,Zs=n(),W=i("p"),sa=Ys(`As the discretized volumetric rendeing equation is fully differentiable, the weights of the underlying neural network can then be trained using a reconstruction loss on the rendered pixels.
Many NeRF approaches use a pixel-wise error term that can be written as follows:
`),Ws=new ta(!1),Fs=n(),F=i("p"),F.textContent=Ca,Ns=n(),N=i("p"),N.innerHTML=xa,Bs=n(),B=i("p"),B.textContent=_a,Vs=n(),V=i("p"),V.textContent=Ia,Gs=n(),E(G.$$.fragment),Hs=n(),H=i("p"),H.textContent=za,$s=n(),$=i("p"),$.innerHTML=ka,Xs=n(),X=i("p"),X.innerHTML=Ra,Ss=n(),E(S.$$.fragment),Qs=n(),Q=i("p"),Q.innerHTML=Za,As=n(),E(A.$$.fragment),Ls=n(),L=i("p"),L.innerHTML=Wa,qs=n(),es=i("p"),this.h()},l(s){const a=Qa("svelte-u9bgzb",document.head);r=p(a,"META",{name:!0,content:!0}),a.forEach(e),is=l(s),as=p(s,"P",{}),ns(as).forEach(e),ps=l(s),Y(c.$$.fragment,s),ms=l(s),h=p(s,"P",{"data-svelte-h":!0}),m(h)!=="svelte-jul4ou"&&(h.innerHTML=ia),rs=l(s),Y(u.$$.fragment,s),os=l(s),y=p(s,"P",{"data-svelte-h":!0}),m(y)!=="svelte-b8xglz"&&(y.innerHTML=pa),cs=l(s),d=p(s,"P",{"data-svelte-h":!0}),m(d)!=="svelte-16j6utc"&&(d.innerHTML=ma),hs=l(s),g=p(s,"P",{"data-svelte-h":!0}),m(g)!=="svelte-1oqw5y8"&&(g.innerHTML=ra),us=l(s),f=p(s,"P",{"data-svelte-h":!0}),m(f)!=="svelte-pma9sc"&&(f.textContent=oa),ys=l(s),Y(v.$$.fragment,s),ds=l(s),b=p(s,"P",{"data-svelte-h":!0}),m(b)!=="svelte-3tnuk1"&&(b.textContent=ca),gs=l(s),w=p(s,"P",{"data-svelte-h":!0}),m(w)!=="svelte-y9lvf0"&&(w.textContent=ha),fs=l(s),M=p(s,"P",{"data-svelte-h":!0}),m(M)!=="svelte-1rmodno"&&(M.innerHTML=ua),vs=l(s),T=p(s,"P",{"data-svelte-h":!0}),m(T)!=="svelte-1ka629u"&&(T.innerHTML=ya),bs=l(s),J=p(s,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1tobfln"&&(J.innerHTML=da),ws=l(s),U=p(s,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1c917er"&&(U.innerHTML=ga),Ms=l(s),j=p(s,"P",{"data-svelte-h":!0}),m(j)!=="svelte-1oxe61n"&&(j.innerHTML=fa),Ts=l(s),C=p(s,"P",{"data-svelte-h":!0}),m(C)!=="svelte-9jqkju"&&(C.innerHTML=va),Js=l(s),x=p(s,"P",{"data-svelte-h":!0}),m(x)!=="svelte-105mi7d"&&(x.textContent=ba),Us=l(s),_=p(s,"P",{"data-svelte-h":!0}),m(_)!=="svelte-8gnilz"&&(_.innerHTML=wa),js=l(s),o=p(s,"P",{});var ts=ns(o);Ds=Ps(ts,`The principles behind the process of volumetric rendering are well established in classical computer graphics pipelines and do not stem from NeRFs.
What is important for the use case of NeRFs is that this step is `),q=p(ts,"STRONG",{"data-svelte-h":!0}),m(q)!=="svelte-tu9yc4"&&(q.textContent=Ma),Ks=Ps(ts,` in order to allow for backpropagation. The simplest form of volumetric rendering in NeRFs can be formulated as follows:
`),Cs=na(ts,!1),ts.forEach(e),xs=l(s),I=p(s,"P",{"data-svelte-h":!0}),m(I)!=="svelte-cxj60j"&&(I.textContent=Ta),_s=l(s),z=p(s,"P",{});var aa=ns(z);Os=Ps(aa,`After discretization, the equation above can be computed as the following sum:
`),Is=na(aa,!1),aa.forEach(e),zs=l(s),k=p(s,"P",{"data-svelte-h":!0}),m(k)!=="svelte-1lchi64"&&(k.textContent=Ja),ks=l(s),R=p(s,"P",{"data-svelte-h":!0}),m(R)!=="svelte-130vc1o"&&(R.innerHTML=Ua),Rs=l(s),Z=p(s,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-1ylz9no"&&(Z.innerHTML=ja),Zs=l(s),W=p(s,"P",{});var ea=ns(W);sa=Ps(ea,`As the discretized volumetric rendeing equation is fully differentiable, the weights of the underlying neural network can then be trained using a reconstruction loss on the rendered pixels.
Many NeRF approaches use a pixel-wise error term that can be written as follows:
`),Ws=na(ea,!1),ea.forEach(e),Fs=l(s),F=p(s,"P",{"data-svelte-h":!0}),m(F)!=="svelte-qt58wn"&&(F.textContent=Ca),Ns=l(s),N=p(s,"P",{"data-svelte-h":!0}),m(N)!=="svelte-1fogha9"&&(N.innerHTML=xa),Bs=l(s),B=p(s,"P",{"data-svelte-h":!0}),m(B)!=="svelte-900j4b"&&(B.textContent=_a),Vs=l(s),V=p(s,"P",{"data-svelte-h":!0}),m(V)!=="svelte-wakkft"&&(V.textContent=Ia),Gs=l(s),Y(G.$$.fragment,s),Hs=l(s),H=p(s,"P",{"data-svelte-h":!0}),m(H)!=="svelte-v8crjg"&&(H.textContent=za),$s=l(s),$=p(s,"P",{"data-svelte-h":!0}),m($)!=="svelte-cyuxuu"&&($.innerHTML=ka),Xs=l(s),X=p(s,"P",{"data-svelte-h":!0}),m(X)!=="svelte-92cgsm"&&(X.innerHTML=Ra),Ss=l(s),Y(S.$$.fragment,s),Qs=l(s),Q=p(s,"P",{"data-svelte-h":!0}),m(Q)!=="svelte-zhe5l7"&&(Q.innerHTML=Za),As=l(s),Y(A.$$.fragment,s),Ls=l(s),L=p(s,"P",{"data-svelte-h":!0}),m(L)!=="svelte-z2v2y8"&&(L.innerHTML=Wa),qs=l(s),es=p(s,"P",{}),ns(es).forEach(e),this.h()},h(){Fa(r,"name","hf:doc:metadata"),Fa(r,"content",qa),Cs.a=null,Is.a=null,Ws.a=null},m(s,a){P(document.head,r),t(s,is,a),t(s,as,a),t(s,ps,a),D(c,s,a),t(s,ms,a),t(s,h,a),t(s,rs,a),D(u,s,a),t(s,os,a),t(s,y,a),t(s,cs,a),t(s,d,a),t(s,hs,a),t(s,g,a),t(s,us,a),t(s,f,a),t(s,ys,a),D(v,s,a),t(s,ds,a),t(s,b,a),t(s,gs,a),t(s,w,a),t(s,fs,a),t(s,M,a),t(s,vs,a),t(s,T,a),t(s,bs,a),t(s,J,a),t(s,ws,a),t(s,U,a),t(s,Ms,a),t(s,j,a),t(s,Ts,a),t(s,C,a),t(s,Js,a),t(s,x,a),t(s,Us,a),t(s,_,a),t(s,js,a),t(s,o,a),P(o,Ds),P(o,q),P(o,Ks),Cs.m(Na,o),t(s,xs,a),t(s,I,a),t(s,_s,a),t(s,z,a),P(z,Os),Is.m(Ba,z),t(s,zs,a),t(s,k,a),t(s,ks,a),t(s,R,a),t(s,Rs,a),t(s,Z,a),t(s,Zs,a),t(s,W,a),P(W,sa),Ws.m(Va,W),t(s,Fs,a),t(s,F,a),t(s,Ns,a),t(s,N,a),t(s,Bs,a),t(s,B,a),t(s,Vs,a),t(s,V,a),t(s,Gs,a),D(G,s,a),t(s,Hs,a),t(s,H,a),t(s,$s,a),t(s,$,a),t(s,Xs,a),t(s,X,a),t(s,Ss,a),D(S,s,a),t(s,Qs,a),t(s,Q,a),t(s,As,a),D(A,s,a),t(s,Ls,a),t(s,L,a),t(s,qs,a),t(s,es,a),Es=!0},p:Ha,i(s){Es||(K(c.$$.fragment,s),K(u.$$.fragment,s),K(v.$$.fragment,s),K(G.$$.fragment,s),K(S.$$.fragment,s),K(A.$$.fragment,s),Es=!0)},o(s){O(c.$$.fragment,s),O(u.$$.fragment,s),O(v.$$.fragment,s),O(G.$$.fragment,s),O(S.$$.fragment,s),O(A.$$.fragment,s),Es=!1},d(s){s&&(e(is),e(as),e(ps),e(ms),e(h),e(rs),e(os),e(y),e(cs),e(d),e(hs),e(g),e(us),e(f),e(ys),e(ds),e(b),e(gs),e(w),e(fs),e(M),e(vs),e(T),e(bs),e(J),e(ws),e(U),e(Ms),e(j),e(Ts),e(C),e(Js),e(x),e(Us),e(_),e(js),e(o),e(xs),e(I),e(_s),e(z),e(zs),e(k),e(ks),e(R),e(Rs),e(Z),e(Zs),e(W),e(Fs),e(F),e(Ns),e(N),e(Bs),e(B),e(Vs),e(V),e(Gs),e(Hs),e(H),e($s),e($),e(Xs),e(X),e(Ss),e(Qs),e(Q),e(As),e(Ls),e(L),e(qs),e(es)),e(r),ss(c,s),ss(u,s),ss(v,s),ss(G,s),ss(S,s),ss(A,s)}}}const qa='{"title":"Neural Radiance Fields (NeRFs)","local":"neural-radiance-fields-nerfs","sections":[{"title":"Short History 📖","local":"short-history-","sections":[],"depth":2},{"title":"Underlying approach (Vanilla NeRF) 📘🔍","local":"underlying-approach-vanilla-nerf-","sections":[],"depth":2},{"title":"Train your own NeRF","local":"train-your-own-nerf","sections":[],"depth":2},{"title":"Current advancements in the field","local":"sota","sections":[],"depth":2}],"depth":1}';function Ea(la){return $a(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Oa extends Xa{constructor(r){super(),Sa(this,r,Ea,La,Ga,{})}}export{Oa as component};
