[RFC 140] Simple package paths, part 1b: Enabling the directory structure

.github/CODEOWNERS

@@ -49,6 +49,9 @@
 
 # pkgs/by-name
 /pkgs/test/nixpkgs-check-by-name @infinisil
+/pkgs/by-name/README.md @infinisil
+/pkgs/top-level/by-name-overlay.nix @infinisil
+/.github/workflows/check-by-name.nix @infinisil
 
 # Nixpkgs build-support
 /pkgs/build-support/writers @lassulus @Profpatsch

.github/workflows/check-by-name.yml

@@ -0,0 +1,49 @@
+# Checks pkgs/by-name (see pkgs/by-name/README.md)
+# using the nixpkgs-check-by-name tool (see pkgs/test/nixpkgs-check-by-name)
+name: Check pkgs/by-name
+
+# The pre-built tool is fetched from a channel,
+# making it work predictable on all PRs
+on: pull_request
+
+# The tool doesn't need any permissions, it only outputs success or not based on the checkout
+permissions: {}
+
+jobs:
+  check:
+    # This is x86_64-linux, for which the tool is always prebuilt on the nixos-* channels,
+    # as specified in nixos/release-combined.nix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v22
+      - name: Determining channel to use for dependencies
+        run: |
+          echo "Determining which channel to use for PR base branch $GITHUB_BASE_REF"
+          if [[ "$GITHUB_BASE_REF" =~ ^(release|staging|staging-next)-([0-9][0-9]\.[0-9][0-9])$ ]]; then
+              # Use the release channel for all PRs to release-XX.YY, staging-XX.YY and staging-next-XX.YY
+              channel=nixos-${BASH_REMATCH[2]}
+              echo "PR is for a release branch, using release channel $channel"
+          else
+              # Use the nixos-unstable channel for all other PRs
+              channel=nixos-unstable
+              echo "PR is for a non-release branch, using unstable channel $channel"
+          fi
+          echo "channel=$channel" >> "$GITHUB_ENV"
+      - name: Fetching latest version of channel
+        run: |
+          echo "Fetching latest version of channel $channel"
+          # This is probably the easiest way to get Nix to output the path to a downloaded channel!
+          nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=channel:"$channel")
+          # This file only exists in channels
+          rev=$(<"$nixpkgs"/.git-revision)
+          echo "Channel $channel is at revision $rev"
+          echo "nixpkgs=$nixpkgs" >> "$GITHUB_ENV"
+          echo "rev=$rev" >> "$GITHUB_ENV"
+      - name: Fetching pre-built nixpkgs-check-by-name from the channel
+        run: |
+          echo "Fetching pre-built nixpkgs-check-by-name from channel $channel at revision $rev"
+          # Passing --max-jobs 0 makes sure that we won't build anything
+          nix-build "$nixpkgs" -A tests.nixpkgs-check-by-name --max-jobs 0
+      - name: Running nixpkgs-check-by-name
+        run: result/bin/nixpkgs-check-by-name .

pkgs/README.md

@@ -7,7 +7,7 @@ See the [CONTRIBUTING.md](../CONTRIBUTING.md) document for more general informat
 
 - [`top-level`](./top-level): Entrypoints, package set aggregations
   - [`impure.nix`](./top-level/impure.nix), [`default.nix`](./top-level/default.nix), [`config.nix`](./top-level/config.nix): Definitions for the evaluation entry point of `import <nixpkgs>`
-  - [`stage.nix`](./top-level/stage.nix), [`all-packages.nix`](./top-level/all-packages.nix), [`splice.nix`](./top-level/splice.nix): Definitions for the top-level attribute set made available through `import <nixpkgs> {…}`
+  - [`stage.nix`](./top-level/stage.nix), [`all-packages.nix`](./top-level/all-packages.nix), [`by-name-overlay.nix`](./top-level/by-name-overlay.nix), [`splice.nix`](./top-level/splice.nix): Definitions for the top-level attribute set made available through `import <nixpkgs> {…}`
   - `*-packages.nix`, [`linux-kernels.nix`](./top-level/linux-kernels.nix), [`unixtools.nix`](./top-level/unixtools.nix): Aggregations of nested package sets defined in `development`
   - [`aliases.nix`](./top-level/aliases.nix), [`python-aliases.nix`](./top-level/python-aliases.nix): Aliases for package definitions that have been renamed or removed
   - `release*.nix`, [`make-tarball.nix`](./top-level/make-tarball.nix), [`packages-config.nix`](./top-level/packages-config.nix), [`metrics.nix`](./top-level/metrics.nix), [`nixpkgs-basic-release-checks.nix`](./top-level/nixpkgs-basic-release-checks.nix): Entry-points and utilities used by Hydra for continuous integration
@@ -19,6 +19,7 @@ See the [CONTRIBUTING.md](../CONTRIBUTING.md) document for more general informat
 - [`stdenv`](./stdenv): [Standard environment](https://nixos.org/manual/nixpkgs/stable/#part-stdenv)
 - [`pkgs-lib`](./pkgs-lib): Definitions for utilities that need packages but are not needed for packages
 - [`test`](./test): Tests not directly associated with any specific packages
+- [`by-name`](./by-name): Top-level packages organised by name ([docs](./by-name/README.md))
 - All other directories loosely categorise top-level packages definitions, see [category hierarchy][categories]
 
 ## Quick Start to Adding a Package
@@ -49,22 +50,25 @@ Now that this is out of the way. To add a package to Nixpkgs:
    $ cd nixpkgs
    ```
 
-2. Find a good place in the Nixpkgs tree to add the Nix expression for your package. For instance, a library package typically goes into `pkgs/development/libraries/pkgname`, while a web browser goes into `pkgs/applications/networking/browsers/pkgname`. See the [category hierarchy section][categories] for some hints on the tree organisation. Create a directory for your package, e.g.
+2. Create a package directory `pkgs/by-name/so/some-package` where `some-package` is the package name and `so` is the lowercased 2-letter prefix of the package name:
 
    ```ShellSession
-   $ mkdir pkgs/development/libraries/libfoo
+   $ mkdir -p pkgs/by-name/so/some-package
    ```
 
-3. In the package directory, create a Nix expression — a piece of code that describes how to build the package. In this case, it should be a _function_ that is called with the package dependencies as arguments, and returns a build of the package in the Nix store. The expression should usually be called `default.nix`.
+   For more detailed information, see [here](./by-name/README.md).
+
+3. Create a `package.nix` file in the package directory, containing a Nix expression — a piece of code that describes how to build the package. In this case, it should be a _function_ that is called with the package dependencies as arguments, and returns a build of the package in the Nix store.
 
    ```ShellSession
-   $ emacs pkgs/development/libraries/libfoo/default.nix
-   $ git add pkgs/development/libraries/libfoo/default.nix
+   $ emacs pkgs/by-name/so/some-package/package.nix
+   $ git add pkgs/by-name/so/some-package/package.nix
    ```
 
-   You can have a look at the existing Nix expressions under `pkgs/` to see how it’s done. Here are some good ones:
+   You can have a look at the existing Nix expressions under `pkgs/` to see how it’s done, some of which are also using the [category hierarchy](#category-hierarchy).
+   Here are some good ones:
 
-   - GNU Hello: [`pkgs/applications/misc/hello/default.nix`](applications/misc/hello/default.nix). Trivial package, which specifies some `meta` attributes which is good practice.
+   - GNU Hello: [`pkgs/by-name/he/hello/package.nix`](./by-name/he/hello/package.nix). Trivial package, which specifies some `meta` attributes which is good practice.
 
    - GNU cpio: [`pkgs/tools/archivers/cpio/default.nix`](tools/archivers/cpio/default.nix). Also a simple package. The generic builder in `stdenv` does everything for you. It has no dependencies beyond `stdenv`.
 
@@ -94,21 +98,13 @@ Now that this is out of the way. To add a package to Nixpkgs:
 
    The exact syntax and semantics of the Nix expression language, including the built-in function, are [described in the Nix manual](https://nixos.org/manual/nix/stable/language/).
 
-4. Add a call to the function defined in the previous step to [`pkgs/top-level/all-packages.nix`](top-level/all-packages.nix) with some descriptive name for the variable, e.g. `libfoo`.
-
-   ```ShellSession
-   $ emacs pkgs/top-level/all-packages.nix
-   ```
-
-   The attributes in that file are sorted by category (like “Development / Libraries”) that more-or-less correspond to the directory structure of Nixpkgs, and then by attribute name.
-
 5. To test whether the package builds, run the following command from the root of the nixpkgs source tree:
 
    ```ShellSession
-   $ nix-build -A libfoo
+   $ nix-build -A some-package
    ```
 
-   where `libfoo` should be the variable name defined in the previous step. You may want to add the flag `-K` to keep the temporary build directory in case something fails. If the build succeeds, a symlink `./result` to the package in the Nix store is created.
+   where `some-package` should be the package name. You may want to add the flag `-K` to keep the temporary build directory in case something fails. If the build succeeds, a symlink `./result` to the package in the Nix store is created.
 
 6. If you want to install the package into your profile (optional), do
 
@@ -121,9 +117,19 @@ Now that this is out of the way. To add a package to Nixpkgs:
 ## Category Hierarchy
 [categories]: #category-hierarchy
 
-Each package should be stored in its own directory somewhere in the `pkgs/` tree, i.e. in `pkgs/category/subcategory/.../pkgname`. Below are some rules for picking the right category for a package. Many packages fall under several categories; what matters is the _primary_ purpose of a package. For example, the `libxml2` package builds both a library and some tools; but it’s a library foremost, so it goes under `pkgs/development/libraries`.
+Most top-level packages are organised in a loosely-categorised directory hierarchy in this directory.
+See the [overview](#overview) for which directories are part of this.
+
+This category hierarchy is partially deprecated and will be migrated away over time.
+The new `pkgs/by-name` directory ([docs](./by-name/README.md)) should be preferred instead.
+The category hierarchy may still be used for packages that should be imported using an alternate `callPackage`, such as `python3Packages.callPackage` or `libsForQt5.callPackage`.
 
-When in doubt, consider refactoring the `pkgs/` tree, e.g. creating new categories or splitting up an existing category.
+If that is the case for a new package, here are some rules for picking the right category.
+Many packages fall under several categories; what matters is the _primary_ purpose of a package.
+For example, the `libxml2` package builds both a library and some tools; but it’s a library foremost, so it goes under `pkgs/development/libraries`.
+
+<details>
+<summary>Categories</summary>
 
 **If it’s used to support _software development_:**
 
@@ -299,6 +305,8 @@ A (typically large) program with a distinct user interface, primarily used inter
 
 - `misc`
 
+</details>
+
 # Conventions
 
 ## Package naming

pkgs/by-name/README.md

@@ -0,0 +1,101 @@
+# Name-based package directories
+
+The structure of this directory maps almost directly to top-level package attributes.
+This is the recommended way to add new top-level packages to Nixpkgs [when possible](#limitations).
+
+## Example
+
+The top-level package `pkgs.some-package` may be declared by setting up this file structure:
+
+```
+pkgs
+└── by-name
+   ├── so
+   ┊  ├── some-package
+      ┊  └── package.nix
+
+```
+
+Where `some-package` is the package name and `so` is the lowercased 2-letter prefix of the package name.
+
+The `package.nix` may look like this:
+
+```nix
+# A function taking an attribute set as an argument
+{
+  # Get access to top-level attributes for use as dependencies
+  lib,
+  stdenv,
+  libbar,
+
+  # Make this derivation configurable using `.override { enableBar = true }`
+  enableBar ? false,
+}:
+
+# The return value must be a derivation
+stdenv.mkDerivation {
+  # ...
+  buildInputs =
+    lib.optional enableBar libbar;
+}
+```
+
+You can also split up the package definition into more files in the same directory if necessary.
+
+Once defined, the package can be built from the Nixpkgs root directory using:
+```
+nix-build -A some-package
+```
+
+See the [general package conventions](../README.md#conventions) for more information on package definitions.
+
+### Changing implicit attribute defaults
+
+The above expression is called using these arguments by default:
+```nix
+{
+  lib = pkgs.lib;
+  stdenv = pkgs.stdenv;
+  libbar = pkgs.libbar;
+}
+```
+
+But the package might need `pkgs.libbar_2` instead.
+While the function could be changed to take `libbar_2` directly as an argument,
+this would change the `.override` interface, breaking code like `.override { libbar = ...; }`.
+So instead it is preferable to use the same generic parameter name `libbar`
+and override its value in [`pkgs/top-level/all-packages.nix`](../top-level/all-packages.nix):
+
+```nix
+libfoo = callPackage ../by-name/so/somePackage/package.nix {
+  libbar = libbar_2;
+};
+```
+
+## Limitations
+
+There's some limitations as to which packages can be defined using this structure:
+
+- Only packages defined using `pkgs.callPackage`.
+  This excludes packages defined using `pkgs.python3Packages.callPackage ...`.
+
+  Instead use the [category hierarchy](../README.md#category-hierarchy) for such attributes.
+
+- Only top-level packages.
+  This excludes packages for other package sets like `pkgs.pythonPackages.*`.
+
+  Refer to the definition and documentation of the respective package set to figure out how such packages can be declared.
+
+## Validation
+
+CI performs [certain checks](../test/nixpkgs-check-by-name/README.md#validity-checks) on the `pkgs/by-name` structure.
+This is done using the [`nixpkgs-check-by-name` tool](../test/nixpkgs-check-by-name).
+The version of this tool used is the one that corresponds to the NixOS channel of the PR base branch.
+See [here](../../.github/workflows/check-by-name.yml) for details.
+
+The tool can be run locally using
+
+```bash
+nix-build -A tests.nixpkgs-check-by-name
+result/bin/nixpkgs-check-by-name .
+```

pkgs/test/nixpkgs-check-by-name/README.md

@@ -1,11 +1,12 @@
 # Nixpkgs pkgs/by-name checker
 
 This directory implements a program to check the [validity](#validity-checks) of the `pkgs/by-name` Nixpkgs directory once introduced.
+It is being used by [this GitHub Actions workflow](../../../.github/workflows/check-by-name.yml).
 This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pull/140).
 
 ## API
 
-This API may be changed over time if the CI making use of it is adjusted to deal with the change appropriately, see [Hydra builds](#hydra-builds).
+This API may be changed over time if the CI workflow making use of it is adjusted to deal with the change appropriately.
 
 - Command line: `nixpkgs-check-by-name <NIXPKGS>`
 - Arguments:

pkgs/top-level/all-packages.nix

@@ -32514,8 +32514,6 @@ with pkgs;
 
   heimer = libsForQt5.callPackage ../applications/misc/heimer { };
 
-  hello = callPackage ../applications/misc/hello { };
-
   hello-wayland = callPackage ../applications/graphics/hello-wayland { };
 
   hello-unfree = callPackage ../applications/misc/hello-unfree { };

pkgs/top-level/by-name-overlay.nix

@@ -0,0 +1,50 @@
+# This file turns the pkgs/by-name directory (see its README.md for more info) into an overlay that adds all the defined packages.
+# No validity checks are done here,
+# instead this file is optimised for performance,
+# and validity checks are done by CI on PRs.
+
+# Type: Path -> Overlay
+baseDirectory:
+let
+  # Because of Nix's import-value cache, importing lib is free
+  lib = import ../../lib;
+
+  inherit (builtins)
+    readDir
+    ;
+
+  inherit (lib.attrsets)
+    mapAttrs
+    mapAttrsToList
+    mergeAttrsList
+    ;
+
+  # Package files for a single shard
+  # Type: String -> String -> AttrsOf Path
+  namesForShard = shard: type:
+    if type != "directory" then
+      # Ignore all non-directories. Technically only README.md is allowed as a file in the base directory, so we could alternatively:
+      # - Assume that README.md is the only file and change the condition to `shard == "README.md"` for a minor performance improvement.
+      #   This would however cause very poor error messages if there's other files.
+      # - Ensure that README.md is the only file, throwing a better error message if that's not the case.
+      #   However this would make for a poor code architecture, because one type of error would have to be duplicated in the validity checks and here.
+      # Additionally in either of those alternatives, we would have to duplicate the hardcoding of "README.md"
+      { }
+    else
+      mapAttrs
+        (name: _: baseDirectory + "/${shard}/${name}/package.nix")
+        (readDir (baseDirectory + "/${shard}"));
+
+  # The attribute set mapping names to the package files defining them
+  # This is defined up here in order to allow reuse of the value (it's kind of expensive to compute)
+  # if the overlay has to be applied multiple times
+  packageFiles = mergeAttrsList (mapAttrsToList namesForShard (readDir baseDirectory));
+in
+# TODO: Consider optimising this using `builtins.deepSeq packageFiles`,
+# which could free up the above thunks and reduce GC times.
+# Currently this would be hard to measure until we have more packages
+# and ideally https://github.com/NixOS/nix/pull/8895
+self: super:
+mapAttrs (name: file:
+  self.callPackage file { }
+) packageFiles

pkgs/top-level/stage.nix

@@ -8,6 +8,13 @@
    arguments. Normal users should not import this directly but instead
    import `pkgs/default.nix` or `default.nix`. */
 
+let
+  # An overlay to auto-call packages in ../by-name.
+  # By defining it at the top of the file,
+  # this value gets reused even if this file is imported multiple times,
+  # thanks to Nix's import-value cache.
+  autoCalledPackages = import ./by-name-overlay.nix ../by-name;
+in
 
 { ## Misc parameters kept the same for all stages
   ##
@@ -279,6 +286,7 @@ let
     stdenvAdapters
     trivialBuilders
     splice
+    autoCalledPackages
     allPackages
     otherPackageSets
     aliases