kernels/pr_552/en/_app/immutable/nodes/8.2e9f8295.js

import{s as xe,n as we,o as ve}from"../chunks/scheduler.f3b1e791.js";import{S as Te,i as ke,e as s,s as n,c as d,h as Ue,a as o,d as i,b as a,f as Me,g as c,j as r,k as ge,l as Ce,m as l,n as m,t as u,o as h,p as y}from"../chunks/index.023a9934.js";import{C as $e}from"../chunks/CopyLLMTxtMenu.db780d33.js";import{C as _e}from"../chunks/CodeBlock.c625e428.js";import{H as W,E as Je}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.86b6e38e.js";function Be(se){let p,Z,L,Q,b,G,f,P,M,Y,g,oe=`kernel-builder uses a Nix-based builder that orchestrates the build. The Nix
builder provides:`,H,x,re=`<li>Reproducible evaluation. The same Nix builder version will always produce
the same derivations (build recipes).</li> <li>Largely reproducible builds by using a build sandbox that only has the
dependencies specified in a derivation.</li> <li>Seamless creation of different build environments (e.g. different Torch
and CUDA combinations).</li>`,F,w,A,v,pe="A kernel derivation builds a kernel in the following steps:",X,T,de=`<li>Generate CMake files for the kernel using
<code>kernel-builder create-pyproject</code>.</li> <li>Generate Ninja build files using CMake.</li> <li>Build the kernel using Ninja.</li> <li>Perform various checks on the compiled kernel, such as:
<ul><li>Verify that the kernel only uses ABI3/<code>manylinux_2_28</code> symbols.</li> <li>Verify that the kernel can be loaded by the <code>kernels</code> Python package.</li></ul></li> <li>Strip runpaths (ELF-embedded library directories) from kernel binaries
to make the kernel distribution-independent.</li>`,R,k,V,U,ce=`To achieve <code>manylinux_2_28</code> compatibility, kernels are built using a
toolchain similar to the <code>manylinux_2_28</code> Docker images. This toolchain
is based on the gcc toolsets from AlmaLinux 8. <code>manylinux_2_28</code> <a href="https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based" rel="nofollow">uses
AlmaLinux 8 as its base</a>,
so we have to compile against the same glibc/libstdc++ versions to
ensure compatibility.`,z,C,me=`We repackage the AlmaLinux 8 toolsets and libstdc++ as Nix derivations (see
the <code>nix-builder/packages/manylinux_2_28</code> source directory). Then we merge
various toolset packages to an unwrapped gcc that resembles unwrapped gcc in
nixpkgs. Finally, we wrap binutils and gcc to combine them into a stdenv.`,K,$,ue=`The stdenv does not reuse glibc from AlmaLinux, since its dynamic loader has
hardcoded FHS paths (<code>/lib64</code> etc.) that are not valid in Nix. Using this
dynamic loader results in linking errors, since the paths in the dynamic
loader are used as a last resort (to link glibc libraries). So, instead we
build our own glibc 2.28 package and use that.`,D,_,O,J,he=`We repackage various existing package sets as Nix derivations. For instance,
this is done for ROCm, XPU, and manylinux_2_28 packages. These package sets
all follow the same pattern:`,q,B,ee,j,ye=`We use a fixed point to build up the package set as a list of
<a href="https://nixos.org/manual/nixpkgs/stable/#sec-overlays-definition" rel="nofollow">overlays</a>.
This has various benefits. For instance, it allows us to refine the
package set incrementally and we can refer to the final versions of
packages in intermediate overlays.`,te,E,be="The package sets all use a similar list of overlays:",ie,I,fe=`<li>An initial overlay (<code>components.nix</code>) that applies a generic builder
to the package set metadata. The metadata typically comes from a Yum/DNF
repository that contains RPM packages.The generic builder will extract the
RPMs and move binaries, libraries, and headers to the right location. This
results in a set of Nix derivations that may or may not build.</li> <li>The next overlay (<code>overrides.nix</code>) fixes up derivations created by the
generic builder that do not build. Fixing the derivations typically consists
of adding missing dependencies and changing embedded FHS paths to Nix store
paths.</li> <li>Additional overlays with derivations that combine outputs from previous
overlays. One typical example are derivations that construct a full compiler
toolchain.</li>`,le,N,ne,S,ae;return b=new $e({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new W({props:{title:"Nix Builder design",local:"nix-builder-design",headingTag:"h1"}}),M=new W({props:{title:"Introduction",local:"introduction",headingTag:"h2"}}),w=new W({props:{title:"Kernel build steps",local:"kernel-build-steps",headingTag:"h2"}}),k=new W({props:{title:"manylinux_2_28 compatibility",local:"manylinux228-compatibility",headingTag:"h2"}}),_=new W({props:{title:"The package set pattern",local:"the-package-set-pattern",headingTag:"h2"}}),B=new _e({props:{code:"JTdCJTBBJTIwJTIwbGliJTJDJTBBJTIwJTIwY2FsbFBhY2thZ2UlMkMlMEElMjAlMjBuZXdTY29wZSUyQyUwQSUyMCUyMHBrZ3MlMkMlMEElN0QlM0ElMEElMEElN0IlMEElMjAlMjBwYWNrYWdlTWV0YWRhdGElMkMlMEElN0QlM0ElMEElMEFsZXQlMEElMjAlMjBpbmhlcml0JTIwKGxpYi5maXhlZFBvaW50cyklMjBleHRlbmRzJTIwY29tcG9zZU1hbnlFeHRlbnNpb25zJTNCJTBBJTBBJTIwJTIwZml4ZWRQb2ludCUyMCUzRCUyMGZpbmFsJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwaW5oZXJpdCUyMGxpYiUzQiUwQSUyMCUyMCU3RCUzQiUwQSUyMCUyMGNvbXBvc2VkJTIwJTNEJTIwbGliLmNvbXBvc2VNYW55RXh0ZW5zaW9ucyUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMyUyMEJhc2UlMjBwYWNrYWdlJTIwc2V0LiUwQSUyMCUyMCUyMCUyMChpbXBvcnQlMjAuJTJGY29tcG9uZW50cy5uaXglMjAlN0IlMjBpbmhlcml0JTIwcGFja2FnZU1ldGFkYXRhJTNCJTIwJTdEKSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMFBhY2thZ2Utc3BlY2lmaWMlMjBvdmVycmlkZXMuJTBBJTIwJTIwJTIwJTIwKGltcG9ydCUyMC4lMkZvdmVycmlkZXMubml4KSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMEFkZGl0aW9uYWwlMjBvdmVybGF5cyUyMHRoYXQlMjBleHRlbmQlMjB0aGUlMjBwYWNrYWdlJTIwc2V0LiUwQSUyMCUyMCUyMCUyMChpbXBvcnQlMjAuJTJGc29tZS1vdmVybGF5Lm5peCklMEElMjAlMjAlNUQlM0IlMEFpbiUwQWxpYi5tYWtlU2NvcGUlMjBuZXdTY29wZSUyMChsaWIuZXh0ZW5kcyUyMGNvbXBvc2VkJTIwZml4ZWRQb2ludCk=",highlighted:`{
  lib,
  callPackage,
  newScope,
  pkgs,
}:

{
  packageMetadata,
}:

<span class="hljs-keyword">let</span>
  <span class="hljs-keyword">inherit</span> (lib.fixedPoints) extends composeManyExtensions;

  <span class="hljs-attr">fixedPoint</span> = final: {
    <span class="hljs-keyword">inherit</span> lib;
  };
  <span class="hljs-attr">composed</span> = lib.composeManyExtensions [
    <span class="hljs-comment"># Base package set.</span>
    (<span class="hljs-built_in">import</span> ./components.nix { <span class="hljs-keyword">inherit</span> packageMetadata; })

    <span class="hljs-comment"># Package-specific overrides.</span>
    (<span class="hljs-built_in">import</span> ./overrides.nix)

    <span class="hljs-comment"># Additional overlays that extend the package set.</span>
    (<span class="hljs-built_in">import</span> ./some-overlay.nix)
  ];
<span class="hljs-keyword">in</span>
lib.makeScope newScope (lib.extends composed fixedPoint)`,lang:"nix",wrap:!1}}),N=new Je({props:{source:"https://github.com/huggingface/kernels/blob/main/docs/source/builder/design-nix-builder.md"}}),{c(){p=s("meta"),Z=n(),L=s("p"),Q=n(),d(b.$$.fragment),G=n(),d(f.$$.fragment),P=n(),d(M.$$.fragment),Y=n(),g=s("p"),g.textContent=oe,H=n(),x=s("ul"),x.innerHTML=re,F=n(),d(w.$$.fragment),A=n(),v=s("p"),v.textContent=pe,X=n(),T=s("ol"),T.innerHTML=de,R=n(),d(k.$$.fragment),V=n(),U=s("p"),U.innerHTML=ce,z=n(),C=s("p"),C.innerHTML=me,K=n(),$=s("p"),$.innerHTML=ue,D=n(),d(_.$$.fragment),O=n(),J=s("p"),J.textContent=he,q=n(),d(B.$$.fragment),ee=n(),j=s("p"),j.innerHTML=ye,te=n(),E=s("p"),E.textContent=be,ie=n(),I=s("ul"),I.innerHTML=fe,le=n(),d(N.$$.fragment),ne=n(),S=s("p"),this.h()},l(e){const t=Ue("svelte-u9bgzb",document.head);p=o(t,"META",{name:!0,content:!0}),t.forEach(i),Z=a(e),L=o(e,"P",{}),Me(L).forEach(i),Q=a(e),c(b.$$.fragment,e),G=a(e),c(f.$$.fragment,e),P=a(e),c(M.$$.fragment,e),Y=a(e),g=o(e,"P",{"data-svelte-h":!0}),r(g)!=="svelte-10m3qhc"&&(g.textContent=oe),H=a(e),x=o(e,"UL",{"data-svelte-h":!0}),r(x)!=="svelte-k9b528"&&(x.innerHTML=re),F=a(e),c(w.$$.fragment,e),A=a(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-wtbtic"&&(v.textContent=pe),X=a(e),T=o(e,"OL",{"data-svelte-h":!0}),r(T)!=="svelte-18gvsag"&&(T.innerHTML=de),R=a(e),c(k.$$.fragment,e),V=a(e),U=o(e,"P",{"data-svelte-h":!0}),r(U)!=="svelte-dg7mnn"&&(U.innerHTML=ce),z=a(e),C=o(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-1wrujd7"&&(C.innerHTML=me),K=a(e),$=o(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-rn6k4c"&&($.innerHTML=ue),D=a(e),c(_.$$.fragment,e),O=a(e),J=o(e,"P",{"data-svelte-h":!0}),r(J)!=="svelte-13648ym"&&(J.textContent=he),q=a(e),c(B.$$.fragment,e),ee=a(e),j=o(e,"P",{"data-svelte-h":!0}),r(j)!=="svelte-11ywwcl"&&(j.innerHTML=ye),te=a(e),E=o(e,"P",{"data-svelte-h":!0}),r(E)!=="svelte-1o8pfoq"&&(E.textContent=be),ie=a(e),I=o(e,"UL",{"data-svelte-h":!0}),r(I)!=="svelte-fpgu5k"&&(I.innerHTML=fe),le=a(e),c(N.$$.fragment,e),ne=a(e),S=o(e,"P",{}),Me(S).forEach(i),this.h()},h(){ge(p,"name","hf:doc:metadata"),ge(p,"content",je)},m(e,t){Ce(document.head,p),l(e,Z,t),l(e,L,t),l(e,Q,t),m(b,e,t),l(e,G,t),m(f,e,t),l(e,P,t),m(M,e,t),l(e,Y,t),l(e,g,t),l(e,H,t),l(e,x,t),l(e,F,t),m(w,e,t),l(e,A,t),l(e,v,t),l(e,X,t),l(e,T,t),l(e,R,t),m(k,e,t),l(e,V,t),l(e,U,t),l(e,z,t),l(e,C,t),l(e,K,t),l(e,$,t),l(e,D,t),m(_,e,t),l(e,O,t),l(e,J,t),l(e,q,t),m(B,e,t),l(e,ee,t),l(e,j,t),l(e,te,t),l(e,E,t),l(e,ie,t),l(e,I,t),l(e,le,t),m(N,e,t),l(e,ne,t),l(e,S,t),ae=!0},p:we,i(e){ae||(u(b.$$.fragment,e),u(f.$$.fragment,e),u(M.$$.fragment,e),u(w.$$.fragment,e),u(k.$$.fragment,e),u(_.$$.fragment,e),u(B.$$.fragment,e),u(N.$$.fragment,e),ae=!0)},o(e){h(b.$$.fragment,e),h(f.$$.fragment,e),h(M.$$.fragment,e),h(w.$$.fragment,e),h(k.$$.fragment,e),h(_.$$.fragment,e),h(B.$$.fragment,e),h(N.$$.fragment,e),ae=!1},d(e){e&&(i(Z),i(L),i(Q),i(G),i(P),i(Y),i(g),i(H),i(x),i(F),i(A),i(v),i(X),i(T),i(R),i(V),i(U),i(z),i(C),i(K),i($),i(D),i(O),i(J),i(q),i(ee),i(j),i(te),i(E),i(ie),i(I),i(le),i(ne),i(S)),i(p),y(b,e),y(f,e),y(M,e),y(w,e),y(k,e),y(_,e),y(B,e),y(N,e)}}}const je='{"title":"Nix Builder design","local":"nix-builder-design","sections":[{"title":"Introduction","local":"introduction","sections":[],"depth":2},{"title":"Kernel build steps","local":"kernel-build-steps","sections":[],"depth":2},{"title":"manylinux_2_28 compatibility","local":"manylinux228-compatibility","sections":[],"depth":2},{"title":"The package set pattern","local":"the-package-set-pattern","sections":[],"depth":2}],"depth":1}';function Ee(se){return ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ze extends Te{constructor(p){super(),ke(this,p,Ee,Be,xe,{})}}export{Ze as component};