Segmentation fault

[Defman]

I'm not sure if this is the right place to post the issue or if it should be on the main nixos repository.

Describe the issue

Running nix build or nix flake check causes a segmentation fault.

Additional context

Tried on the following machines.

• OS X Arm64
• GitHub ubuntu runner

echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
echo "sandbox = false" >> /etc/nix/nix.conf
echo "filter-syscalls = false" >> /etc/nix/nix.conf

default.nix/shell.nix/flake.nix:

{
  description = "Application packaged using poetry2nix";

  inputs.flake-utils.url = "github:numtide/flake-utils";
  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
  inputs.poetry2nix = {
    url = "github:nix-community/poetry2nix";
    inputs.nixpkgs.follows = "nixpkgs";
  };

  outputs = { self, nixpkgs, flake-utils, poetry2nix }:
    flake-utils.lib.eachDefaultSystem (system:
      let
        # see https://github.com/nix-community/poetry2nix/tree/master#api for more functions and examples.
        inherit (poetry2nix.legacyPackages.${system}) mkPoetryEnv defaultPoetryOverrides;
        pkgs = nixpkgs.legacyPackages.${system};
        pypkgs-build-requirements = {
          django-autoslug = [ "setuptools" ];
          django-hosts = [ "setuptools" ];
          django-autocomplete-light = [ "setuptools" ];
          django-bootstrap3 = [ "setuptools" ];
          django-classy-tags = [ "setuptools" ];
          crispy-bootstrap3 = [ "setuptools" ];
          django-easy-pdf3 = [ "setuptools" ];
          
        };
        p2n-overrides = defaultPoetryOverrides.extend (self: super:
          builtins.mapAttrs (package: build-requirements:
            (builtins.getAttr package super).overridePythonAttrs (old: {
              buildInputs = (old.buildInputs or [ ]) ++ (builtins.map (pkg: if builtins.isString pkg then builtins.getAttr pkg super else pkg) build-requirements);
            })
          ) pypkgs-build-requirements
        );
      in
      {
        packages = {
          myapp = mkPoetryEnv { 
            projectDir = self; 
            overrides = p2n-overrides;
            groups = [];
            checkGroups = [];
            python = pkgs.python310;
            extras = [];
          };
          default = self.packages.${system}.myapp;
        };

        devShells.default = pkgs.mkShell {
          packages = [ poetry2nix.packages.${system}.poetry ];
        };
      });
}

flake.lock:

{
  "nodes": {
    "flake-utils": {
      "inputs": {
        "systems": "systems"
      },
      "locked": {
        "lastModified": 1692799911,
        "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=",
        "owner": "numtide",
        "repo": "flake-utils",
        "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "flake-utils",
        "type": "github"
      }
    },
    "flake-utils_2": {
      "inputs": {
        "systems": "systems_2"
      },
      "locked": {
        "lastModified": 1689068808,
        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
        "owner": "numtide",
        "repo": "flake-utils",
        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "flake-utils",
        "type": "github"
      }
    },
    "nix-github-actions": {
      "inputs": {
        "nixpkgs": [
          "poetry2nix",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1688870561,
        "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=",
        "owner": "nix-community",
        "repo": "nix-github-actions",
        "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
        "repo": "nix-github-actions",
        "type": "github"
      }
    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1693250523,
        "narHash": "sha256-y3up5gXMTbnCsXrNEB5j+7TVantDLUYyQLu/ueiXuyg=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "3efb0f6f404ec8dae31bdb1a9b17705ce0d6986e",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixos-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "poetry2nix": {
      "inputs": {
        "flake-utils": "flake-utils_2",
        "nix-github-actions": "nix-github-actions",
        "nixpkgs": [
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1693051011,
        "narHash": "sha256-HNbuVCS/Fnl1YZOjBk9/MlIem+wM8fvIzTH0CVQrLSQ=",
        "owner": "nix-community",
        "repo": "poetry2nix",
        "rev": "5b3a5151cf212021ff8d424f215fb030e4ff2837",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
        "repo": "poetry2nix",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
        "flake-utils": "flake-utils",
        "nixpkgs": "nixpkgs",
        "poetry2nix": "poetry2nix"
      }
    },
    "systems": {
      "locked": {
        "lastModified": 1681028828,
        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
        "owner": "nix-systems",
        "repo": "default",
        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
        "type": "github"
      },
      "original": {
        "owner": "nix-systems",
        "repo": "default",
        "type": "github"
      }
    },
    "systems_2": {
      "locked": {
        "lastModified": 1681028828,
        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
        "owner": "nix-systems",
        "repo": "default",
        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
        "type": "github"
      },
      "original": {
        "owner": "nix-systems",
        "repo": "default",
        "type": "github"
      }
    }
  },
  "root": "root",
  "version": 7
}

pyproject.toml:

[tool.poetry]
name = "unf-dk"
version = "0.1.0"
description = ""
authors = ["Your Name <[email protected]>"]
readme = "README.md"
packages = [{include = "unf"}]

[tool.poetry.dependencies]
python = "~3.10"
Django = "~4.2"
django-extensions = "~3.2"
django-reversion = "~5.0"
djangorestframework= "~3.14"
django-filter = "~23.2"
django-classy-tags = "~4.0"
django-cors-headers = "~3.14"
django-bootstrap3 = "~23.1"
django-crispy-forms = "~2.0"
django-formtools = "~2.4"
django-autocomplete-light = "~3.9"
django-widget-tweaks = "~1.4"
django-maintenance-mode = "~0.18"
django-haystack = "~3.2"
pygments = "~2.15"
whoosh = "~2.7"
django-phonenumber-field = "~7.1"
phonenumbers = "~8.13"
markdown = "~3.4"
python-markdown-math = "~0.8"
bleach = "~6.0"
django-easy-pdf3 = "~0.1"
reportlab = "~4.0"
fabric = "~3.0"
progressbar2 = "~4.2"
pytz = "2023.3"
Pillow = "~10.0"
python-dateutil = "~2.8"
pycountry = "~22.3.5"
six = "~1.16"
requests = "~2.29"
html2text = "~2020.1"
passlib = "~1.7"
google-api-python-client = "~2.86"
google-auth-oauthlib = "~1.0"
django-autoslug = "~1.9"
psycopg2 = "~2.9"
easy-thumbnails = "~2.8"
crispy-bootstrap3 = "^2022.1"
django-hosts = "^5.2"
gunicorn = "^21.2.0"
django-environ = "^0.10.0"

[tool.poetry.group.dev.dependencies]
django-debug-toolbar = "~4.0"
django-appconf = "~1.0"
ruff = "~0.0"
types-pytz = "~2023.3"
types-PyYAML = "~6.0"
types-simplejson = "3.19.0.0"
django-stubs = { extras = ["compatible-mypy"], version = "~4.2" }
django-stubs-ext = "~4.2"
types-six = "~1.16.21.7"
types-Markdown = "~3.4.2.5"
types-bleach = "~6.0.0.1"
types-python-dateutil = "~2.8.19.10"
types-requests = "~2.29"
types-urllib3 = "~1.26.25.8"

[tool.poetry.scripts]
init = "runserver:init"
side = "runserver:side"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.ruff]
# Exclude a variety of commonly ignored directories.
exclude = [
    "**/migrations",
    "config/settings",
]

# Same as Black.
line-length = 120

# Assume Python 3.10.
target-version = "py310"

[tool.ruff.per-file-ignores]
"__init__.py" = ["E402", "F401", "F403"]

[tool.mypy]
mypy_path = "./unf"
plugins = ["mypy_django_plugin.main"]

strict_optional = true
ignore_missing_imports = true

[tool.django-stubs]
django_settings_module = "config.settings"

[[tool.mypy.overrides]]
module = "config.settings.*"
ignore_errors = true

poetry.lock

[jcollie]

I'm not getting a segmentation fault, but I can reliably get the following error with poetry2nix from this repository. The error goes away and the build succeeds if I use poetry2nix from nixpkgs.

evaluating derivation 'git+file:///home/jeff/dev/greendeck#greendeck'error: stack overflow (possible infinite recursion)

[apeschar]

I get this when I make poetry2nix follow nixpkgs/nixos-unstable, but not with nixpkgs/nixos-23.05 or the default version used by poetry2nix (without inputs.nixpkgs.follows).

[Janik-Haag]

With the current rev of nixpkgs-unstable and poetry2nix I get:

evaluating derivation 'git+file:///home/janik/Projects/usbguard-applet#packages.x86_64-linux.default'[1]    51388 invalid system call (core dumped)  nix build

when it runs long enough and it consumes about 16gig of memory before doing so
linux/nixos x86-64

[Defman]

I should maybe add that I have the following cofigured.

echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
echo "sandbox = false" >> /etc/nix/nix.conf
echo "filter-syscalls = false" >> /etc/nix/nix.conf

[KiruyaMomochi]

I got it when trying to nix shell after nix flake init --template github:nix-community/poetry2nix. Looking at results from nix develop --debug, I found it evaluating the current directory over and over again.

As said by @apeschar, everything works fine when using nixpkgs/nixos-23.05. Therefore, although a bit of silly, I bisected nixpkgs from that ref to latest nightly.

It's caused by NixOS/nixpkgs#248866 which switches to PyPA build, installer, and flit-core to build Python packages. The following commits are the root cause:

• NixOS/nixpkgs@3cd71e0: python3.pkgs.wheel: 0.38.4 -> 0.41.1
  In this version, the build backend of wheel changes from setuptools to flit-core. However, our overrides/build-systems.json does not change. This is the first half.

• NixOS/nixpkgs@6b63908: poetry2nix.overrides.wheel: adapt to new wheel version
  This commit modified overrides/default.nix in nixpkg's poetry2nix. But this does affect our own poetry2nix.

• NixOS/nixpkgs@253a291: poetry2nix.overrides: update build systems for bootstrap packages

  We need to clarify these or else the wrong build system gets added and
  we find ourselves with an infinite recursion.

  Here the infinite recursion has been resolved. The problem is the same as the last commit - this is only changed in nixpkg's poetry2nix. The one in our repository is still broken.

• NixOS/nixpkgs@5a9dda2: python3.pkgs.setuptools: build without bootstrapped-pip
  Setuptools is built with wheel instead of bootstrapped-pip. This is the other half.

As our overrides only have the knowledge about old nixpkgs, it still think wheel depends on setuptools. But now setuptools also depends on wheel because of the changed setuptools/default.nix, and the infinite recursion occurs.

Nixpkgs has fixed this problem between these two commits, and there is actually a PR for us #1290 - although it has not merged yet when I'm writing this.

Currently, the workaround includes using @K900's patch branch, using the poetry2nix from nixpkgs, or use a commit of nixpkgs prior to NixOS/nixpkgs#248866.

[K900]

#1290 is not actually a complete fix, there are still cases where it causes infinite recursion, and I'm pretty sure the correct way of doing it is to just rebuild everything with the nixpkgs bootstrap chain as-is, which would require more invasive changes to poetry2nix.

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/how-to-troubleshoot-an-infinite-recursion-error/32565/5

[vitvakatu]

I still have a segmentation fault using a patched branch. Any advice?

https://github.com/vitvakatu/bleak-nix/blob/main/flake.nix

[K900]

As stated, not a complete fix.

[phelian]

For me things started working when i reverted to quite the old version for poetry alone

inputs = {
  nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
  nixpkgs_poetry.url = "github:NixOS/nixpkgs/release-22.11";
  flake-utils.url = "github:numtide/flake-utils";
  poetry2nix.url = "github:K900/poetry2nix/new-bootstrap-fixes";
};

outputs = { self, nixpkgs, poetry2nix, nixpkgs_poetry, flake-utils }:
  flake-utils.lib.eachDefaultSystem (system:
    let
      cdkOverlay = ...
    
    pkgs = import nixpkgs { inherit system; overlays = [ cdkOverlay  ]; };
    pkgs_poetry = nixpkgs_poetry.legacyPackages.${system};
    
    python = pkgs_poetry.python311;
    
    inherit (poetry2nix.legacyPackages.${system})
      mkPoetryApplication mkPoetryEnv;
    
    poetryEnv = mkPoetryEnv {
      projectDir = ./.;
      preferWheels = true;
      inherit python;
    };
    
    in {
      devShells.default = pkgs.mkShell {
        packages = [
          poetry2nix.packages.${system}.poetry
          poetryEnv
    
          pkgs.pyright
          pkgs.ruff
        ];
      };
    });

The combination of K900's fix and using v22 solved it for me, using nixpkgs 23 introduced some python 3.11 issues
Hope this helps someone

[Dessix]

For anyone searching for a commit from just before the break that is cached by Hydra, I initially tried 9bc7f06a91347a5a7d8c3cb15fc7f27bcc53bd2d (the parent commit before the merge) but found myself building 5 separate versions of LLVM and Clang, the Linux kernel, and other fun CPU-burners.

By manually scrounging through Hydra's job logs, I think I've found 3476a10478587dec90acb14ec6bde0966c545cc0 run by Hydra in build 231993738 as our best bet for a cached build from before the break.

As stated, not a complete fix.

If there's not a fix available, and nobody else knows how to build one just yet, is there a procedure for taking the break back out of main until it's ready to move forward? (This has been noted as an impossibility in discussions with the maintainers.)

Edit: On a related note, does anyone know why this can even cause a segfault? Should that be its own bug on Nix? Maybe alongside some diagnostics for use in debugging infinite recursion, like Haskell has?

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/nix-consumes-all-available-memory-evaluating-a-flake-using-recent-nixpkgs/33322/2

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/nix-consumes-all-available-memory-evaluating-a-flake-using-recent-nixpkgs/33322/4

[takeda]

@FRidh since you made these changes in nixpkgs, do you know what would be needed for poetry2nix to work with new bootstrap? Seeing attempt in #1290 to fix it then being closed is very discouraging :( This looks above my skills.

[FRidh]

Nixpkgs Python now has a small separate set with bootstrapping packages along with new hooks replacing bootstrapped-pip and corresponding hooks. buildPythonPackage checks whether the package to build is a package that is used for bootstrapping and will then update the hooks.

Bootstrapping for the Python core packages had to be changed because packages also started using different build systems. I think it is probably better for poetry2nix to also fix a bootstrapping set of packages just so you know it works.

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/nix-shell-is-eating-up-all-my-32-gb-memory/34134/2

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/function-anonymous-lambda-called-with-unexpected-argument/34267/2

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/function-anonymous-lambda-called-with-unexpected-argument/34267/3