Experimental python bindings

flake.nix

@@ -331,7 +331,7 @@
             name = "nix-${version}";
             inherit version;
 
-            src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self;
+            src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" "python/*" "flake.nix" ] self;
             VERSION_SUFFIX = versionSuffix;
 
             outputs = [ "out" "dev" "doc" ];
@@ -439,6 +439,8 @@
               postUnpack = "sourceRoot=$sourceRoot/perl";
             });
 
+            passthru.python-bindings = prev.callPackage ./python { };
+
             meta.platforms = lib.platforms.unix;
           });
 
@@ -500,6 +502,10 @@
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
+        pythonBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings);
+        # TODO: recurseIntoAttrs or combine multiple tests into a single one
+        pythonBindingsTests = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings.tests.example-buildPythonApplication);
+
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
@@ -645,6 +651,7 @@
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
+        pythonBindings = self.hydraJobs.pythonBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
@@ -727,6 +734,7 @@
             (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
+              python = self.packages.${system}.nix.python-bindings.shell;
             }
         );
   };

python/.gitignore

@@ -0,0 +1,3 @@
+build
+# For clang-tools
+.cache

python/Makefile

@@ -0,0 +1,49 @@
+# This Makefile is only used for development of the Python bindings, it is not
+# used in the Nix build.
+# The reason this exists is to make it easier to develop the Python bindings in
+# tandem with the main Nix.
+# The default `make` (defaults to `make build`) calls the main Nix projects
+# `make install` before calling the Python bindings' `meson compile`, therefore
+# ensuring that the needed Nix dynamic libraries are up-to-date
+
+builddir=build
+
+.PHONY: build
+build: nix-install setup-done
+	meson compile -C $(builddir)
+
+.PHONY: test
+test: nix-install setup-done
+	meson test -C $(builddir) -v
+
+.PHONY: clean
+clean:
+	rm -rf $(builddir)
+
+# We include the main Nix projects Makefile.config file to know the $(libdir)
+# variable, which is where Nix is installed in, which we can then use to setup
+# the meson build
+include ../Makefile.config
+
+# We need the file to exist though
+../Makefile.config:
+	@# Throw a good error message in case ./configure hasn't been run yet
+	@[[ -e ../config.status ]] || ( echo "The main Nix project needs to be configured first, see https://nixos.org/manual/nix/stable/contributing/hacking.html" && exit 1 )
+	@# If ./configure is done, we can create the file ourselves
+	$(MAKE) -C .. Makefile.config
+
+.PHONY: setup
+setup: nix-install
+	@# Make meson be able to find the locally-installed Nix
+	PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH meson setup $(builddir)
+
+.PHONY: setup-done
+setup-done:
+	@# A better error message in case the build directory doesn't exist yet
+	@[[ -e $(builddir) ]] || ( echo "Run 'make setup' once to configure the project build directory" && exit 1 )
+
+.PHONY: nix-install
+nix-install:
+	@# The python bindings don't technically need an _entire_ Nix installation,
+	@# but it seems non-trivial to pick out only exactly the files it actually needs
+	$(MAKE) -C .. install

python/README.md

@@ -0,0 +1,13 @@
+# Python Bindings
+
+This directory contains experimental Python bindings to a small subset of Nix's functionality. These bindings are very fast since they link to the necessary dynamic libraries directly, without having to call the Nix CLI for every operation.
+
+Thanks to [@Mic92](https://github.com/Mic92) who wrote [Pythonix](https://github.com/Mic92/pythonix) which these bindings were originally based on, before they became the official bindings that are part of the Nix project. They were upstreamed to decrease maintenance overhead and make sure they are always up-to-date.
+
+Note that the Python bindings are new and experimental. The interface is likely to change based on known issues and user feedback.
+
+## Documentation
+
+See [index.md](./doc/index.md), which is also rendered in the HTML manual.
+
+To hack on these bindings, see [hacking.md](./doc/hacking.md), also rendered in the HTML manual.

python/default.nix

@@ -0,0 +1,97 @@
+{
+  system,
+  lib,
+  python3,
+  boost,
+  gdb,
+  clang-tools,
+  pkg-config,
+  ninja,
+  meson,
+  nix,
+  mkShell,
+  enableDebugging,
+  recurseIntoAttrs,
+  isShell ? false,
+}:
+let
+  _python = python3;
+in
+let
+  python3 = _python.override { self = enableDebugging _python; };
+  # Extracts tests/init.sh
+  testScripts = nix.overrideAttrs (old: {
+    name = "nix-test-scripts-${old.version}";
+    outputs = [ "out" ];
+    separateDebugInfo = false;
+    buildPhase = ''
+      make tests/{init.sh,common/vars-and-functions.sh}
+    '';
+    script = ''
+      pushd ${placeholder "out"}/libexec >/dev/null
+      source init.sh
+      popd >/dev/null
+    '';
+    passAsFile = [ "script" ];
+    installPhase = ''
+      rm -rf "$out"
+      mkdir -p "$out"/{libexec/common,share/bash}
+      cp tests/init.sh "$out"/libexec
+      cp tests/common/vars-and-functions.sh "$out"/libexec/common
+
+      cp "$scriptPath" "$out"/share/bash/nix-test.sh
+    '';
+    dontFixup = true;
+  });
+in
+python3.pkgs.buildPythonPackage {
+  name = "nix";
+  format = "other";
+
+  src = builtins.path {
+    path = ./.;
+    filter = path: type:
+      path == toString ./meson.build
+      || path == toString ./tests.py
+      || path == toString ./test.sh
+      || lib.hasPrefix (toString ./src) path;
+  };
+
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    ninja
+    pkg-config
+    (meson.override { inherit python3; })
+  ] ++ lib.optional (!isShell) nix;
+
+  buildInputs = nix.propagatedBuildInputs ++ [
+    boost
+  ] ++ lib.optional (!isShell) nix;
+
+  mesonBuildType = "release";
+
+  doInstallCheck = true;
+  TEST_SCRIPTS = testScripts;
+  installCheckPhase = "meson test -v";
+
+  passthru = {
+    exampleEnv = python3.withPackages (p: [ nix.python-bindings ]);
+    tests = {
+      example-buildPythonApplication = import ./examples/buildPythonApplication {
+        inherit nix system testScripts python3;
+      };
+    };
+    shell = mkShell {
+      packages = [
+        clang-tools
+        gdb
+      ];
+      TEST_SCRIPTS = testScripts;
+      inputsFrom = [
+        (nix.python-bindings.override { isShell = true; })
+      ];
+    };
+  };
+}

python/doc/api.md

@@ -0,0 +1,18 @@
+# Experimental Python Bindings
+
+## callExprString
+
+```python
+nix.callExprString(expression: str, arg)
+```
+Parse a nix expression, then call it as a nix function.
+
+Note that this function is experimental and subject to change based on known issues and feedback.
+
+**Parameters:**,
+    `expression` (str): The string containing a nix expression.
+    `arg`: the argument to pass to the function
+
+**Returns:**
+    `result`: the result of the function invocation, converted to python datatypes.
+

python/doc/hacking.md

@@ -0,0 +1,3 @@
+# Python Bindings Hacking
+
+This is how to hack on the bindings

python/doc/index.md

@@ -0,0 +1,32 @@
+# Experimental Python Bindings
+
+Nix comes with minimal experimental Python bindings that link directly to the necessary dynamic libraries, making them very fast.
+
+## Trying it out
+
+The easiest way to try out the bindings is using the provided example environment:
+
+```
+$ nix run github:NixOS/nix#nix.python-bindings.exampleEnv
+Python 3.10.8 (main, Oct 11 2022, 11:35:05) [GCC 11.3.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import nix
+>>> nix.callExprString('"Hello ${name}!"', arg={"name": "Python"}))
+'Hello Python!'
+```
+
+For the available functions and their interfaces, see the API section.
+
+## Build integration
+
+In the future these Python bindings will be available from Nixpkgs as `python3Packages.nix`.
+
+Until then the Python bindings are only available from the Nix derivation via the `python-bindings` [passthru attribute](https://nixos.org/manual/nixpkgs/stable/#var-stdenv-passthru). Without any modifications, this derivation is built for the default Python 3 version from the Nixpkgs version used to build Nix. This Python version might not match the Python version of the project you're trying to use them in. Therefore it is recommended to override the bindings with the correct Python version using
+
+```
+nix.python-bindings.override {
+  python = myPythonVersion;
+}
+```
+
+For complete examples, see https://github.com/NixOS/nix/tree/master/python/examples

python/examples/buildPythonApplication/default.nix

@@ -0,0 +1,37 @@
+{
+  system ? builtins.currentSystem,
+  pkgs ? import (fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/545c7a31e5dedea4a6d372712a18e00ce097d462.tar.gz";
+    sha256 = "1dbsi2ccq8x0hyl8n0hisigj8q19amvj9irzfbgy4b3szb6x2y6l";
+  }) {
+    config = {};
+    overlays = [];
+    inherit system;
+  },
+  python3 ? pkgs.python3,
+  nix ? (import ../../..).default,
+  testScripts,
+}:
+let
+  nixBindings = nix.python-bindings.override { inherit python3; };
+in python3.pkgs.buildPythonApplication {
+  pname = "hello-nix";
+  version = "0.1";
+  src = builtins.path {
+    path = ./.;
+    filter = path: type:
+      pkgs.lib.hasPrefix (toString ./hello) path
+      || path == toString ./setup.py;
+  };
+  propagatedBuildInputs = [ nixBindings ];
+  doInstallCheck = true;
+  nativeCheckInputs = [
+    nix
+  ];
+  installCheckPhase = ''
+    (
+      source ${testScripts}/share/bash/nix-test.sh
+      $out/bin/hello-nix
+    )
+  '';
+}

python/examples/buildPythonApplication/hello/__init__.py

@@ -0,0 +1,4 @@
+import nix
+
+def greet():
+    print("Evaluating 1 + 1 in Nix gives: " + str(nix.callExprString("_: 1 + 1", None)))

python/examples/buildPythonApplication/setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="hello-nix",
+    version="0.1",
+    packages=find_packages(),
+    entry_points={
+        'console_scripts': [
+            'hello-nix = hello:greet',
+        ],
+    },
+)

python/meson.build

@@ -0,0 +1,17 @@
+project('python-nix', 'cpp',
+  version : '0.1.8',
+  license : 'LGPL-2.0',
+)
+
+python_mod = import('python')
+py_installation = python_mod.find_installation()
+
+nix_expr_dep = dependency('nix-expr', required: true)
+
+subdir('src')
+
+fs = import('fs')
+
+env = environment()
+env.prepend('PYTHONPATH', fs.parent(nix_bindings.full_path()))
+test('python test', find_program('bash'), args : files('test.sh'), env : env)