build: Add hacks function for 2-way Nixpkgs Python infrastructure

This adds a `hacks.toNixpkgs` that generates a Nixpkgs overlay from pyproject.nix build (uv2nix) packages.
It is conceptually similar to `hacks.nixpkgsPrebuilt`, just in reverse.

Author: adisbladis

build/checks/build-systems.nix

@@ -688,6 +688,74 @@ let
             setuptools = [ ];
           };
         };
+
+      hatch-vcs =
+        {
+          stdenv,
+          python3Packages,
+          pyprojectHook,
+          resolveBuildSystem,
+        }:
+        stdenv.mkDerivation {
+          inherit (python3Packages.hatch-vcs)
+            pname
+            version
+            src
+            meta
+            ;
+
+          nativeBuildInputs = [
+            pyprojectHook
+          ]
+          ++ resolveBuildSystem {
+            hatchling = [ ];
+          };
+        };
+
+      # Package urllib3 for testing hacks.toNixpkgs.
+      # This is a simple dependency that we can test both dependencies & optional-dependencies with.
+      urllib3 =
+        {
+          stdenv,
+          python3Packages,
+          pyprojectHook,
+          resolveBuildSystem,
+        }:
+        stdenv.mkDerivation {
+          inherit (python3Packages.urllib3)
+            pname
+            version
+            src
+            meta
+            ;
+
+          passthru = {
+            optional-dependencies = {
+              brotli = {
+                brotli = [ ];
+              };
+              zstd = {
+                zstandard = [ ];
+              };
+              socks = {
+                PySocks = [ ];
+              };
+              h2 = {
+                h2 = [ ];
+              };
+            };
+          };
+
+          nativeBuildInputs = [
+            pyprojectHook
+          ]
+          ++ resolveBuildSystem {
+            hatchling = [ ];
+            hatch-vcs = [ ];
+            setuptools-scm = [ ];
+          };
+        };
+
     };
 
   crossOverlay = lib.composeExtensions (_final: prev: {

build/checks/default.nix

@@ -52,6 +52,12 @@ let
     in
 
     {
+      setuptools-scm = pythonSet.setuptools-scm.overrideAttrs (old: {
+        passthru = old.passthru // {
+          inherit pythonSet;
+        };
+      });
+
       make-venv =
         pkgs.runCommand "venv-run-build-test"
           {

build/hacks/checks.nix

@@ -102,4 +102,30 @@ in
       ln -s ${venv} $out
     '';
 
+  toNixpkgs =
+    let
+      overlay = hacks.toNixpkgs {
+        inherit pythonSet;
+        packages = [
+          "pip" # Testing dependencies
+          "urllib3" # Testing optional-dependencies
+        ];
+      };
+
+      python = pkgs.python3.override {
+        packageOverrides = overlay;
+        self = python;
+      };
+
+      pythonEnv = python.withPackages (ps: [
+        ps.urllib3
+        ps.pip
+      ]);
+    in
+    assert pkgs.python3.pkgs.urllib3 != python.pkgs.urllib3;
+    assert pkgs.python3.pkgs.pip != python.pkgs.pip;
+    pkgs.runCommand "toNixpkgs-check" { } ''
+      ${pythonEnv}/bin/python -c 'import urllib3'
+      ${pythonEnv}/bin/pip --version > $out
+    '';
 }

build/hacks/default.nix

@@ -2,7 +2,17 @@
 
 let
   inherit (pkgs) stdenv;
-  inherit (lib) isDerivation isAttrs;
+  inherit (lib) isDerivation isAttrs listToAttrs;
+  inherit (builtins)
+    concatMap
+    elem
+    attrNames
+    mapAttrs
+    isFunction
+    isList
+    typeOf
+    filter
+    ;
 
 in
 {
@@ -192,4 +202,158 @@ in
         rustc
       ];
     });
+
+  /**
+    Create a nixpkgs Python (buildPythonPackage) compatible package from a pyproject.nix build package.
+
+    Adapts a package by:
+    - Activating a wheel output, if not already enabled
+    - Create a package using generated wheel as input
+
+    # Example
+
+    ```nix
+    toNixpkgs {
+      inherit pythonSet;
+      packages = [ "requests" ];
+    }
+    =>
+    «lambda @ /nix/store/f05hjk9fh1m5py5j1ixzly07p4lla56x-source/build/hacks/default.nix:263:5»
+    ```
+
+    # Type
+
+    ```
+    nixpkgsPrebuilt :: AttrSet -> derivation
+    ```
+
+    # Arguments
+
+    pythonSet
+    : Pyproject.nix build Python package set
+
+    packages
+    : List/predicate of overlay member packages
+  */
+  toNixpkgs =
+    let
+      # Always filter out when generating set
+      wellKnown = [
+        "python"
+        "pkgs"
+        "stdenv"
+        "pythonPkgsBuildHost"
+        "resolveBuildSystem"
+        "resolveVirtualEnv"
+        "mkVirtualEnv"
+        "hooks"
+      ];
+    in
+    {
+      pythonSet,
+      packages ? null,
+    }:
+    let
+      packages' =
+        if (packages == null || isFunction packages) then
+          (
+            let
+              hookNames = attrNames pythonSet.hooks;
+              predicate = if packages == null then (_: true) else packages;
+            in
+            filter (name: !elem name wellKnown && !elem name hookNames && predicate name) (attrNames pythonSet)
+          )
+        else if isList packages then
+          packages
+        else
+          throw "Unhandled packages type: ${typeOf packages}";
+
+      # Ensure wheel artifacts are created for all packages we are generating from
+      pythonSet' = pythonSet.overrideScope (
+        _final: prev:
+        listToAttrs (
+          map (
+            name:
+            let
+              drv = prev.${name};
+            in
+            {
+              inherit name;
+              value =
+                if elem "dist" (drv.outputs or [ ]) then
+                  drv
+                else
+                  drv.overrideAttrs (old: {
+                    outputs = (old.outputs or [ "out" ]) ++ [ "dist" ];
+                  });
+            }
+          ) packages'
+        )
+      );
+    in
+    pythonPackagesFinal: _pythonPackagesPrev:
+    let
+      inherit (pythonPackagesFinal) buildPythonPackage pkgs;
+      inherit (pkgs) autoPatchelfHook;
+    in
+    listToAttrs (
+      map (
+        name:
+        let
+          from = pythonSet'.${name};
+          dependencies = from.passthru.dependencies or { };
+          optional-dependencies = from.passthru.optional-dependencies or { };
+        in
+        {
+          inherit name;
+          value = buildPythonPackage {
+            inherit (from) pname version;
+            src = from.dist;
+
+            format = "wheel";
+            dontBuild = true;
+
+            # Default wheelUnpackPhase assumes we are passing a single wheel, but we are passing a dist dir
+            unpackPhase = ''
+              runHook preUnpack
+              mkdir dist
+              cp ${from.dist}/* dist/
+              # runHook postUnpack # Calls find...?
+            '';
+
+            # Include any buildInputs from build for autoPatchelfHook
+            buildInputs = from.buildInputs or [ ];
+
+            nativeBuildInputs = lib.optional stdenv.isLinux [
+              autoPatchelfHook
+            ];
+
+            propagatedBuildInputs = concatMap (
+              name:
+              let
+                pkg = pythonPackagesFinal.${name};
+                extras = dependencies.${name};
+              in
+              [ pkg ] ++ concatMap (extra: pkg.optional-dependencies.${extra}) extras
+            ) (attrNames dependencies);
+
+            passthru = {
+              optional-dependencies = mapAttrs (
+                name: dependencies:
+                concatMap (
+                  name:
+                  let
+                    pkg = pythonPackagesFinal.${name};
+                    extras = dependencies.${name};
+                  in
+                  [ pkg ] ++ concatMap (extra: pkg.optional-dependencies.${extra}) extras
+                ) (attrNames dependencies)
+              ) optional-dependencies;
+            };
+
+            # Note: PEP-735 dependency groups are dropped as nixpkgs lacks support.
+          };
+        }
+      ) packages'
+    );
 }

build/hacks/tests.nix

@@ -0,0 +1,97 @@
+{
+  pkgs,
+  lib,
+  pyproject-nix,
+}:
+let
+  hacks = pkgs.callPackages pyproject-nix.build.hacks { };
+
+  python = pkgs.python3;
+
+  buildSystems = import ../checks/build-systems.nix {
+    inherit lib;
+  };
+
+  pythonSet =
+    (pkgs.callPackage pyproject-nix.build.packages {
+      inherit python;
+    }).overrideScope
+      buildSystems;
+
+in
+{
+  toNixpkgs = {
+    testList =
+      let
+        overlay = hacks.toNixpkgs {
+          inherit pythonSet;
+          packages = [
+            "pip" # Testing dependencies
+            "urllib3" # Testing optional-dependencies
+          ];
+        };
+        python = pkgs.python3.override {
+          packageOverrides = overlay;
+          self = python;
+        };
+
+      in
+      {
+        expr = {
+          urllib3 = python.pkgs.urllib3.version;
+          pip = python.pkgs.pip.version;
+        };
+        expected = {
+          urllib3 = "2.4.0";
+          pip = "25.0.1";
+        };
+      };
+
+    testPredicate =
+      let
+        overlay = hacks.toNixpkgs {
+          inherit pythonSet;
+          packages = lib.flip lib.elem [
+            "pip"
+            "urllib3"
+          ];
+        };
+        python = pkgs.python3.override {
+          packageOverrides = overlay;
+          self = python;
+        };
+      in
+      {
+        expr = {
+          urllib3 = python.pkgs.urllib3.version;
+          pip = python.pkgs.pip.version;
+        };
+        expected = {
+          urllib3 = "2.4.0";
+          pip = "25.0.1";
+        };
+      };
+
+    testNull =
+      let
+        overlay = hacks.toNixpkgs {
+          inherit pythonSet;
+        };
+        python = pkgs.python3.override {
+          packageOverrides = overlay;
+          self = python;
+        };
+
+      in
+      {
+        expr = {
+          urllib3 = python.pkgs.urllib3.version;
+          pip = python.pkgs.pip.version;
+        };
+        expected = {
+          urllib3 = "2.4.0";
+          pip = "25.0.1";
+        };
+      };
+  };
+}