kernels/pr_745/en/_app/immutable/nodes/9.CqLER7DW.js

import"../chunks/DsnmJJEf.js";import{i as g,h as M,C as k,H as a,a as w,E as v,s as x}from"../chunks/CyvF58-O.js";import{p as U,o as f,s as e,f as T,a as h,b as C,c as u,n as J}from"../chunks/DfHjNWj2.js";const j='{"title":"Nix builder design","local":"nix-builder-design","sections":[{"title":"Introduction","local":"introduction","sections":[],"depth":2},{"title":"Kernel build steps","local":"kernel-build-steps","sections":[],"depth":2},{"title":"manylinux_2_28 compatibility","local":"manylinux228-compatibility","sections":[],"depth":2},{"title":"The package set pattern","local":"the-package-set-pattern","sections":[],"depth":2}],"depth":1}';var B=u('<meta name="hf:doc:metadata"/>'),_=u(`<p></p> <!> <!> <!> <p>kernel-builder uses a Nix-based builder that orchestrates the build. The Nix
builder provides:</p> <ul><li>Reproducible evaluation. The same Nix builder version will always produce
the same derivations (build recipes).</li> <li>Largely reproducible builds by using a build sandbox that only has the
dependencies specified in a derivation.</li> <li>Seamless creation of different build environments (e.g. different Torch
and CUDA combinations).</li></ul> <!> <p>A kernel derivation builds a kernel in the following steps:</p> <ol><li>Generate CMake files for the kernel using <code>kernel-builder create-pyproject</code>.</li> <li>Generate Ninja build files using CMake.</li> <li>Build the kernel using Ninja.</li> <li>Perform various checks on the compiled kernel, such as: <ul><li>Verify that the kernel only uses ABI3/<code>manylinux_2_28</code> symbols.</li> <li>Verify that the kernel can be loaded by the <code>kernels</code> Python package.</li></ul></li> <li>Strip runpaths (ELF-embedded library directories) from kernel binaries
to make the kernel distribution-independent.</li></ol> <!> <p>To achieve <code>manylinux_2_28</code> compatibility, kernels are built using a
toolchain similar to the <code>manylinux_2_28</code> Docker images. This toolchain
is based on the gcc toolsets from AlmaLinux 8. <code>manylinux_2_28</code> <a href="https://github.com/pypa/manylinux#manylinux_2_28-almalinux-8-based" rel="nofollow">uses
AlmaLinux 8 as its base</a>,
so we have to compile against the same glibc/libstdc++ versions to
ensure compatibility.</p> <p>We repackage the AlmaLinux 8 toolsets and libstdc++ as Nix derivations (see
the <code>nix-builder/packages/manylinux_2_28</code> source directory). Then we merge
various toolset packages to an unwrapped gcc that resembles unwrapped gcc in
nixpkgs. Finally, we wrap binutils and gcc to combine them into a stdenv.</p> <p>The stdenv does not reuse glibc from AlmaLinux, since its dynamic loader has
hardcoded FHS paths (<code>/lib64</code> etc.) that are not valid in Nix. Using this
dynamic loader results in linking errors, since the paths in the dynamic
loader are used as a last resort (to link glibc libraries). So, instead we
build our own glibc 2.28 package
(see <code>nix-builder/pkgs/manylinux_2_28/stdenv.nix</code>) and use that.</p> <!> <p>We repackage various existing package sets as Nix derivations. For instance,
this is done for ROCm, XPU, and manylinux_2_28 packages. We do this because
we want these libraries to be as close as what the user would install. This
avoids compatibility issues between the kernels and the official vendor
packages. For instance, suppose that we built a ROCm library as a shared
library and ROCm provides the same library as a static library, then compiled
kernels could use symbols that cannot be resolved when installing the official
ROCm packages. Similarly, using the official packages allows us to test
against the official upstream packages.</p> <p>These package sets all follow the same pattern:</p> <!> <p>We use a fixed point to build up the package set as a list of <a href="https://nixos.org/manual/nixpkgs/stable/#sec-overlays-definition" rel="nofollow">overlays</a>.
This has various benefits. For instance, it allows us to refine the
package set incrementally and we can refer to the final versions of
packages in intermediate overlays.</p> <p>The package sets all use a similar list of overlays:</p> <ul><li>An initial overlay (<code>components.nix</code>) that applies a generic builder
to the package set metadata. The metadata typically comes from a Yum/DNF
repository that contains RPM packages.The generic builder will extract the
RPMs and move binaries, libraries, and headers to the right location. This
results in a set of Nix derivations that may or may not build.</li> <li>The next overlay (<code>overrides.nix</code>) fixes up derivations generated by the
generic builder in the previous overlay that do not build. Fixing the
derivations typically consists of adding missing dependencies and changing
embedded FHS paths to Nix store paths.</li> <li>Additional overlays with derivations that combine outputs from previous
overlays. One typical example are derivations that construct a full compiler
toolchain (e.g. <code>nix-builder/pkgs/manylinux_2_28/gcc-unwrapped.nix</code>).</li></ul> <!> <p></p>`,1);function W(m,y){U(y,!1),f(()=>{new URLSearchParams(window.location.search).get("fw")}),g();var i=_();M("foscx3",d=>{var p=B();x(p,"content",j),h(d,p)});var s=e(T(i),2);k(s,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var l=e(s,2);a(l,{title:"Nix builder design",local:"nix-builder-design",headingTag:"h1"});var t=e(l,2);a(t,{title:"Introduction",local:"introduction",headingTag:"h2"});var n=e(t,6);a(n,{title:"Kernel build steps",local:"kernel-build-steps",headingTag:"h2"});var o=e(n,6);a(o,{title:"manylinux_2_28 compatibility",local:"manylinux228-compatibility",headingTag:"h2"});var r=e(o,8);a(r,{title:"The package set pattern",local:"the-package-set-pattern",headingTag:"h2"});var c=e(r,6);w(c,{code:"JTdCJTBBJTIwJTIwbGliJTJDJTBBJTIwJTIwY2FsbFBhY2thZ2UlMkMlMEElMjAlMjBuZXdTY29wZSUyQyUwQSUyMCUyMHBrZ3MlMkMlMEElN0QlM0ElMEElMEElN0IlMEElMjAlMjBwYWNrYWdlTWV0YWRhdGElMkMlMEElN0QlM0ElMEElMEFsZXQlMEElMjAlMjBpbmhlcml0JTIwKGxpYi5maXhlZFBvaW50cyklMjBleHRlbmRzJTIwY29tcG9zZU1hbnlFeHRlbnNpb25zJTNCJTBBJTBBJTIwJTIwZml4ZWRQb2ludCUyMCUzRCUyMGZpbmFsJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwaW5oZXJpdCUyMGxpYiUzQiUwQSUyMCUyMCU3RCUzQiUwQSUyMCUyMGNvbXBvc2VkJTIwJTNEJTIwbGliLmNvbXBvc2VNYW55RXh0ZW5zaW9ucyUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMyUyMEJhc2UlMjBwYWNrYWdlJTIwc2V0LiUwQSUyMCUyMCUyMCUyMChpbXBvcnQlMjAuJTJGY29tcG9uZW50cy5uaXglMjAlN0IlMjBpbmhlcml0JTIwcGFja2FnZU1ldGFkYXRhJTNCJTIwJTdEKSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMFBhY2thZ2Utc3BlY2lmaWMlMjBvdmVycmlkZXMuJTBBJTIwJTIwJTIwJTIwKGltcG9ydCUyMC4lMkZvdmVycmlkZXMubml4KSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMEFkZGl0aW9uYWwlMjBvdmVybGF5cyUyMHRoYXQlMjBleHRlbmQlMjB0aGUlMjBwYWNrYWdlJTIwc2V0LiUwQSUyMCUyMCUyMCUyMChpbXBvcnQlMjAuJTJGc29tZS1vdmVybGF5Lm5peCklMEElMjAlMjAlNUQlM0IlMEFpbiUwQWxpYi5tYWtlU2NvcGUlMjBuZXdTY29wZSUyMChsaWIuZXh0ZW5kcyUyMGNvbXBvc2VkJTIwZml4ZWRQb2ludCk=",highlighted:`{
  lib,
  callPackage,
  newScope,
  pkgs,
}:

{
  packageMetadata,
}:

<span class="hljs-keyword">let</span>
  <span class="hljs-keyword">inherit</span> (lib.fixedPoints) extends composeManyExtensions;

  <span class="hljs-attr">fixedPoint</span> <span class="hljs-operator">=</span> <span class="hljs-params">final:</span> {
    <span class="hljs-keyword">inherit</span> lib;
  };
  <span class="hljs-attr">composed</span> <span class="hljs-operator">=</span> lib.composeManyExtensions [
    <span class="hljs-comment"># Base package set.</span>
    (<span class="hljs-built_in">import</span> <span class="hljs-symbol">./components.nix</span> { <span class="hljs-keyword">inherit</span> packageMetadata; })

    <span class="hljs-comment"># Package-specific overrides.</span>
    (<span class="hljs-built_in">import</span> .<span class="hljs-operator">/</span>overrides.nix)

    <span class="hljs-comment"># Additional overlays that extend the package set.</span>
    (<span class="hljs-built_in">import</span> .<span class="hljs-operator">/</span>some-overlay.nix)
  ];
<span class="hljs-keyword">in</span>
lib.makeScope newScope (lib.extends composed fixedPoint)`,lang:"nix",wrap:!1});var b=e(c,8);v(b,{source:"https://github.com/huggingface/kernels/blob/main/docs/source/builder/design-nix-builder.md"}),J(2),h(m,i),C()}export{W as component};