From 865a71847c9b5a08cfd653f1085ce8c9df18745f Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 16 Apr 2024 11:58:38 -0500 Subject: [PATCH] binlore: migrate override lore to package passthru Lore overrides have been included with binlore's source up to now, but this hasn't worked very well. (It isn't as easy to self-service for people working in nixpkgs, and its use of partial pnames for matching breaks down around some edge cases like version numbers appearing early in perl pnames, or multiple packages having identical pnames.) --- pkgs/by-name/pd/pdf2odt/package.nix | 4 + .../libraries/libarchive/default.nix | 10 ++ .../development/libraries/ncurses/default.nix | 12 ++ pkgs/development/misc/resholve/test.nix | 100 ++++++++++++ .../tools/analysis/binlore/default.nix | 152 ++++++++++++++---- .../linux/nixos-rebuild/default.nix | 9 ++ pkgs/os-specific/linux/procps-ng/default.nix | 9 ++ pkgs/tools/misc/coreutils/default.nix | 16 +- pkgs/tools/misc/linuxquota/default.nix | 6 +- .../tools/nix/nixos-install-tools/default.nix | 7 + pkgs/tools/text/esh/default.nix | 10 +- pkgs/top-level/unixtools.nix | 35 +++- 12 files changed, 338 insertions(+), 32 deletions(-) diff --git a/pkgs/by-name/pd/pdf2odt/package.nix b/pkgs/by-name/pd/pdf2odt/package.nix index eb748da0842be..809af8919343e 100644 --- a/pkgs/by-name/pd/pdf2odt/package.nix +++ b/pkgs/by-name/pd/pdf2odt/package.nix @@ -49,6 +49,10 @@ resholve.mkDerivation rec { imagemagick zip ]; + execer = [ + # zip can exec; confirmed 2 invocations in pdf2odt don't + "cannot:${zip}/bin/zip" + ]; }; meta = with lib; { diff --git a/pkgs/development/libraries/libarchive/default.nix b/pkgs/development/libraries/libarchive/default.nix index d58ba0bc5c5c1..89c224dffbf18 100644 --- a/pkgs/development/libraries/libarchive/default.nix +++ b/pkgs/development/libraries/libarchive/default.nix @@ -23,6 +23,9 @@ , cmake , nix , samba + +# for passthru.lore +, binlore }: assert xarSupport -> libxml2 != null; @@ -124,4 +127,11 @@ stdenv.mkDerivation (finalAttrs: { passthru.tests = { inherit cmake nix samba; }; + + # bsdtar is detected as "cannot" because its exec is internal to + # calls it makes into libarchive itself. If binlore gains support + # for detecting another layer down into libraries, this can be cut. + passthru.lore = (binlore.synthesize finalAttrs.finalPackage '' + execer can bin/bsdtar + ''); }) diff --git a/pkgs/development/libraries/ncurses/default.nix b/pkgs/development/libraries/ncurses/default.nix index 2d383c876cba4..fdff5f4d641a7 100644 --- a/pkgs/development/libraries/ncurses/default.nix +++ b/pkgs/development/libraries/ncurses/default.nix @@ -10,6 +10,7 @@ , mouseSupport ? false, gpm , unicodeSupport ? true , testers +, binlore }: stdenv.mkDerivation (finalAttrs: { @@ -169,6 +170,17 @@ stdenv.mkDerivation (finalAttrs: { rm "$out"/lib/*.a ''; + # I'm not very familiar with ncurses, but it looks like most of the + # exec here will run hard-coded executables. There's one that is + # dynamic, but it looks like it only comes from executing a terminfo + # file, so I think it isn't going to be under user control via CLI? + # Happy to have someone help nail this down in either direction! + # The "capability" is 'iprog', and I could only find 1 real example: + # https://invisible-island.net/ncurses/terminfo.ti.html#tic-linux-s + passthru.lore = (binlore.synthesize ncurses '' + execer cannot bin/{reset,tput,tset} + ''); + meta = with lib; { homepage = "https://www.gnu.org/software/ncurses/"; description = "Free software emulation of curses in SVR4 and more"; diff --git a/pkgs/development/misc/resholve/test.nix b/pkgs/development/misc/resholve/test.nix index d9862bef7cf59..887c4e45cd88d 100644 --- a/pkgs/development/misc/resholve/test.nix +++ b/pkgs/development/misc/resholve/test.nix @@ -25,6 +25,39 @@ , rlwrap , gnutar , bc +# override testing +, esh +, getconf +, libarchive +, locale +, mount +, ncurses +, nixos-install-tools +, nixos-rebuild +, procps +, ps +# known consumers +, aaxtomp3 +, arch-install-scripts +, bashup-events32 +, dgoss +, git-ftp +, ix +, lesspipe +, locate-dominating-file +, mons +, msmtp +, nix-direnv +, pdf2odt +, pdfmm +, rancid +, s0ix-selftest-tool +, unix-privesc-check +, wgnord +, wsl-vpnkit +, xdg-utils +, yadm +, zxfer }: let @@ -190,4 +223,71 @@ rec { echo "Hello" file . ''; + # spot-check lore overrides + loreOverrides = resholve.writeScriptBin "verify-overrides" { + inputs = [ + coreutils + esh + getconf + libarchive + locale + mount + ncurses + procps + ps + ] ++ lib.optionals stdenv.isLinux [ + nixos-install-tools + nixos-rebuild + ]; + interpreter = "none"; + execer = [ + "cannot:${esh}/bin/esh" + ]; + fix = { + mount = true; + }; + } ('' + env b2sum fake args + b2sum fake args + esh fake args + getconf fake args + bsdtar fake args + locale fake args + mount fake args + reset fake args + tput fake args + tset fake args + ps fake args + top fake args + '' + lib.optionalString stdenv.isLinux '' + nixos-generate-config fake args + nixos-rebuild fake args + ''); + + # ensure known consumers in nixpkgs keep working + inherit aaxtomp3; + inherit bashup-events32; + inherit bats; + inherit git-ftp; + inherit ix; + inherit lesspipe; + inherit locate-dominating-file; + inherit mons; + inherit msmtp; + inherit nix-direnv; + inherit pdf2odt; + inherit pdfmm; + inherit shunit2; + inherit xdg-utils; + inherit yadm; +} // lib.optionalAttrs stdenv.isLinux { + inherit arch-install-scripts; + inherit dgoss; + inherit rancid; + inherit unix-privesc-check; + inherit wgnord; + inherit wsl-vpnkit; + inherit zxfer; +} // lib.optionalAttrs (stdenv.isLinux && (stdenv.isi686 || stdenv.isx86_64)) { + inherit s0ix-selftest-tool; } diff --git a/pkgs/development/tools/analysis/binlore/default.nix b/pkgs/development/tools/analysis/binlore/default.nix index e081f43a768fa..d0b172085241e 100644 --- a/pkgs/development/tools/analysis/binlore/default.nix +++ b/pkgs/development/tools/analysis/binlore/default.nix @@ -55,58 +55,156 @@ let # in here, but I'm erring on the side of flexibility # since this form will make it easier to pilot other # uses of binlore. - callback = lore: drv: overrides: '' + callback = lore: drv: '' if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then echo generating binlore for $drv by running: echo "${pkgsBuildBuild.yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${pkgsBuildBuild.yallback}/bin/yallback ${lore.yallback}" else echo "failed to generate binlore for $drv (none of ${drv}/{bin,lib,libexec} exist)" fi - '' + - /* - Override lore for some packages. Unsure, but for now: - 1. start with the ~name (pname-version) - 2. remove characters from the end until we find a match - in overrides/ - 3. execute the override script with the list of expected - lore types - */ - '' - i=''${#identifier} - filter= - while [[ $i > 0 ]] && [[ -z "$filter" ]]; do - if [[ -f "${overrides}/''${identifier:0:$i}" ]]; then - filter="${overrides}/''${identifier:0:$i}" - echo using "${overrides}/''${identifier:0:$i}" to generate overriden binlore for $drv - break - fi - ((i--)) || true # don't break build - done # || true # don't break build + if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then - ${pkgsBuildBuild.yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${pkgsBuildBuild.yallback}/bin/yallback ${lore.yallback} "$filter" + ${pkgsBuildBuild.yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${pkgsBuildBuild.yallback}/bin/yallback ${lore.yallback} fi ''; }; - overrides = (src + "/overrides"); in rec { + /* + Output a directory containing lore for multiple drvs. + + This will `make` lore for drv in drvs and then combine lore + of the same type across all packages into a single file. + + When drvs are also specified in the strip argument, corresponding + lore is made relative by stripping the path of each drv from + matching entries. (This is mainly useful in a build process that + uses a chain of two or more derivations where the output of one + is the source for the next. See resholve for an example.) + */ collect = { lore ? loreDef, drvs, strip ? [ ] }: (runCommand "more-binlore" { } '' mkdir $out for lorefile in ${toString lore.types}; do cat ${lib.concatMapStrings (x: x + "/$lorefile ") (map (make lore) (map lib.getBin (builtins.filter lib.isDerivation drvs)))} > $out/$lorefile - substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace '${x}/' '' ") strip} + substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace-quiet '${x}/' '' ") strip} done ''); - # TODO: echo for debug, can be removed at some point + + /* + Output a directory containing lore for a single drv. + + This produces lore for the derivation (via lore.callback) and + appends any lore that the derivation itself wrote to nix-support + or which was overridden in drv.lore (passthru). + + Since the last entry wins, the effective priority is: + drv.lore > $drv/nix-support > lore generated here by callback + */ make = lore: drv: runCommand "${drv.name}-binlore" { - identifier = drv.name; drv = drv; } ('' mkdir $out touch $out/{${builtins.concatStringsSep "," lore.types}} - ${lore.callback lore drv overrides} + ${lore.callback lore drv} + '' + + # append lore from package's $out and drv.lore (last entry wins) + '' + for lore_type in ${builtins.toString lore.types}; do + if [[ -f "${drv}/nix-support/$lore_type" ]]; then + cat "${drv}/nix-support/$lore_type" >> "$out/$lore_type" + fi + '' + lib.optionalString (builtins.hasAttr "lore" drv) '' + if [[ -f "${drv.lore}/$lore_type" ]]; then + cat "${drv.lore}/$lore_type" >> "$out/$lore_type" + fi + '' + '' + done echo binlore for $drv written to $out ''); + + /* + Utility function for creating override lore for drv. + + In the normal case, we attach this lore to `drv.passthru.lore`. + + The lore argument should be a Shell script (string) that generates + the necessary lore. You can use arbitrary Shell, but this function + includes a shell DSL you can use to declare/generate lore in most + cases. It has the following functions: + + - `execer [...]` + - `wrapper ` + + Writing every override explicitly in a Nix list would be tedious + for large packages, but this small shell DSL enables us to express + many overrides efficiently via pathname expansion/globbing. + + Here's a very general example of both functions: + + passthru.lore = (binlore.synthesize finalAttrs.finalPackage '' + execer can bin/hello bin/{a,b,c} + wrapper bin/hello bin/.hello-wrapped + ''); + + And here's a specific example of how pathname expansion enables us + to express lore for the single-binary variant of coreutils while + being both explicit and (somewhat) efficient: + + passthru = {} // optionalAttrs (singleBinary != false) { + lore = (binlore.synthesize coreutils '' + execer can bin/{chroot,env,install,nice,nohup,runcon,sort,split,stdbuf,timeout} + execer cannot bin/{[,b2sum,base32,base64,basename,basenc,cat,chcon,chgrp,chmod,chown,cksum,comm,cp,csplit,cut,date,dd,df,dir,dircolors,dirname,du,echo,expand,expr,factor,false,fmt,fold,groups,head,hostid,id,join,kill,link,ln,logname,ls,md5sum,mkdir,mkfifo,mknod,mktemp,mv,nl,nproc,numfmt,od,paste,pathchk,pinky,pr,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,shred,shuf,sleep,stat,stty,sum,sync,tac,tail,tee,test,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,vdir,wc,who,whoami,yes} + ''); + }; + + Caution: Be thoughtful about using a bare wildcard (*) glob here. + We should generally override lore only when a human understands if + the executable will exec arbitrary user-passed executables. A bare + glob can match new executables added in future package versions + before anyone can audit them. + */ + synthesize = drv: lore: runCommand "${drv.name}-lore-override" { + drv = drv; + } ('' + execer(){ + local verdict="$1" + + shift + + for path in "$@"; do + if [[ -e "$PWD/$path" ]]; then + echo "$verdict:$PWD/$path" + else + echo "error: Tried to synthesize execer lore for missing file: $PWD/$path" >&2 + exit 2 + fi + done + } >> $out/execers + + wrapper(){ + local wrapper="$1" + local original="$2" + + if [[ ! -e "$wrapper" ]]; then + echo "error: Tried to synthesize wrapper lore for missing wrapper: $PWD/$wrapper" >&2 + exit 2 + fi + + if [[ ! -e "$original" ]]; then + echo "error: Tried to synthesize wrapper lore for missing original: $PWD/$original" >&2 + exit 2 + fi + + echo "$PWD/$wrapper:$PWD/$original" + + } >> $out/wrappers + + mkdir $out + + # lore override commands are relative to the drv root + cd $drv + + '' + lore); } diff --git a/pkgs/os-specific/linux/nixos-rebuild/default.nix b/pkgs/os-specific/linux/nixos-rebuild/default.nix index 17a9bc51732ca..61415ef1145f2 100644 --- a/pkgs/os-specific/linux/nixos-rebuild/default.nix +++ b/pkgs/os-specific/linux/nixos-rebuild/default.nix @@ -10,6 +10,8 @@ , lib , nixosTests , installShellFiles +, binlore +, nixos-rebuild }: let fallback = import ./../../../../nixos/modules/installer/tools/nix-fallback-paths.nix; @@ -49,6 +51,13 @@ substitute { target-host = nixosTests.nixos-rebuild-target-host; }; + # nixos-rebuild can’t execute its arguments + # (but it can run ssh with the with the options stored in $NIX_SSHOPTS, + # and ssh can execute its arguments...) + passthru.lore = (binlore.synthesize nixos-rebuild '' + execer cannot bin/nixos-rebuild + ''); + meta = { description = "Rebuild your NixOS configuration and switch to it, on local hosts and remote."; homepage = "https://github.com/NixOS/nixpkgs/tree/master/pkgs/os-specific/linux/nixos-rebuild"; diff --git a/pkgs/os-specific/linux/procps-ng/default.nix b/pkgs/os-specific/linux/procps-ng/default.nix index e4d245fdc7ce1..96d1fb964e0ed 100644 --- a/pkgs/os-specific/linux/procps-ng/default.nix +++ b/pkgs/os-specific/linux/procps-ng/default.nix @@ -15,6 +15,9 @@ # exception is ‘watch’ which is portable enough to run on pretty much # any UNIX-compatible system. , watchOnly ? !(stdenv.isLinux || stdenv.isCygwin) + +, binlore +, procps }: stdenv.mkDerivation rec { @@ -61,6 +64,12 @@ stdenv.mkDerivation rec { install -m 0644 -D watch.1 $out/share/man/man1/watch.1 ''; + # no obvious exec in documented arguments; haven't trawled source + # to figure out what exec binlore hits on + passthru.lore = (binlore.synthesize procps '' + execer cannot bin/{ps,top,free} + ''); + meta = with lib; { homepage = "https://gitlab.com/procps-ng/procps"; description = "Utilities that give information about processes using the /proc filesystem"; diff --git a/pkgs/tools/misc/coreutils/default.nix b/pkgs/tools/misc/coreutils/default.nix index 388dcd428c717..c7beed838bf37 100644 --- a/pkgs/tools/misc/coreutils/default.nix +++ b/pkgs/tools/misc/coreutils/default.nix @@ -7,6 +7,8 @@ , perl , texinfo , xz +, binlore +, coreutils , gmpSupport ? true, gmp , aclSupport ? stdenv.isLinux, acl , attrSupport ? stdenv.isLinux, attr @@ -27,7 +29,7 @@ assert aclSupport -> acl != null; assert selinuxSupport -> libselinux != null && libsepol != null; let - inherit (lib) concatStringsSep isString optional optionals optionalString; + inherit (lib) concatStringsSep isString optional optionalAttrs optionals optionalString; isCross = (stdenv.hostPlatform != stdenv.buildPlatform); in stdenv.mkDerivation rec { @@ -162,6 +164,18 @@ stdenv.mkDerivation rec { rm -r "$out/share" ''; + passthru = {} // optionalAttrs (singleBinary != false) { + # everything in the single binary gets the same verdict, so we + # override _that case_ with verdicts from separate binaries. + # + # binlore only spots exec in runcon on some platforms (i.e., not + # darwin; I have a note that the behavior may need selinux?) + lore = (binlore.synthesize coreutils '' + execer can bin/{chroot,env,install,nice,nohup,runcon,sort,split,stdbuf,timeout} + execer cannot bin/{[,b2sum,base32,base64,basename,basenc,cat,chcon,chgrp,chmod,chown,cksum,comm,cp,csplit,cut,date,dd,df,dir,dircolors,dirname,du,echo,expand,expr,factor,false,fmt,fold,groups,head,hostid,id,join,kill,link,ln,logname,ls,md5sum,mkdir,mkfifo,mknod,mktemp,mv,nl,nproc,numfmt,od,paste,pathchk,pinky,pr,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,shred,shuf,sleep,stat,stty,sum,sync,tac,tail,tee,test,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,vdir,wc,who,whoami,yes} + ''); + }; + meta = with lib; { homepage = "https://www.gnu.org/software/coreutils/"; description = "The GNU Core Utilities"; diff --git a/pkgs/tools/misc/linuxquota/default.nix b/pkgs/tools/misc/linuxquota/default.nix index b5fc42be82dc7..7a831d6e32764 100644 --- a/pkgs/tools/misc/linuxquota/default.nix +++ b/pkgs/tools/misc/linuxquota/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, e2fsprogs, openldap, pkg-config }: +{ lib, stdenv, fetchurl, e2fsprogs, openldap, pkg-config, binlore, linuxquota }: stdenv.mkDerivation rec { version = "4.09"; @@ -14,6 +14,10 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ pkg-config ]; buildInputs = [ e2fsprogs openldap ]; + passthru.lore = (binlore.synthesize linuxquota '' + execer cannot bin/quota + ''); + meta = with lib; { description = "Tools to manage kernel-level quotas in Linux"; homepage = "https://sourceforge.net/projects/linuxquota/"; diff --git a/pkgs/tools/nix/nixos-install-tools/default.nix b/pkgs/tools/nix/nixos-install-tools/default.nix index e0b24a4e70ddc..365a5fce46f4f 100644 --- a/pkgs/tools/nix/nixos-install-tools/default.nix +++ b/pkgs/tools/nix/nixos-install-tools/default.nix @@ -8,6 +8,7 @@ nixos-install-tools, runCommand, nixosTests, + binlore, }: let inherit (nixos {}) config; @@ -62,6 +63,12 @@ in touch $out ''; }; + + # no documented flags show signs of exec; skim of source suggests + # it's just --help execing man + passthru.lore = (binlore.synthesize nixos-install-tools '' + execer cannot bin/nixos-generate-config + ''); }).overrideAttrs { inherit version; pname = "nixos-install-tools"; diff --git a/pkgs/tools/text/esh/default.nix b/pkgs/tools/text/esh/default.nix index abbeb29234da1..21b13cd79235d 100644 --- a/pkgs/tools/text/esh/default.nix +++ b/pkgs/tools/text/esh/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, asciidoctor, gawk, gnused, runtimeShell }: +{ lib, stdenv, fetchFromGitHub, asciidoctor, gawk, gnused, runtimeShell, binlore, esh }: stdenv.mkDerivation rec { pname = "esh"; @@ -30,6 +30,14 @@ stdenv.mkDerivation rec { doCheck = true; checkTarget = "test"; + # working around a bug in file. Was fixed in + # file 5.41-5.43 but regressed in 5.44+ + # see https://bugs.astron.com/view.php?id=276 + # "can" verdict because of `-s SHELL` arg + passthru.lore = (binlore.synthesize esh '' + execer can bin/esh + ''); + meta = with lib; { description = "Simple templating engine based on shell"; mainProgram = "esh"; diff --git a/pkgs/top-level/unixtools.nix b/pkgs/top-level/unixtools.nix index 454c4802dc915..6e9ef5387e6bf 100644 --- a/pkgs/top-level/unixtools.nix +++ b/pkgs/top-level/unixtools.nix @@ -1,4 +1,4 @@ -{ pkgs, buildEnv, runCommand, lib, stdenv }: +{ pkgs, buildEnv, runCommand, lib, stdenv, binlore }: # These are some unix tools that are commonly included in the /usr/bin # and /usr/sbin directory under more normal distributions. Along with @@ -30,7 +30,9 @@ let priority = 10; platforms = platforms.${stdenv.hostPlatform.parsed.kernel.name} or platforms.all; }; - passthru = { inherit provider; }; + passthru = { inherit provider; } // lib.optionalAttrs (builtins.hasAttr "lore" providers) { + lore = (binlore.synthesize (getBin bins.${cmd}) providers.lore); + }; preferLocalBuild = true; } '' if ! [ -x ${bin} ]; then @@ -75,6 +77,10 @@ let linux = if stdenv.hostPlatform.libc == "glibc" then pkgs.stdenv.cc.libc else pkgs.netbsd.getconf; darwin = pkgs.darwin.system_cmds; + # I don't see any obvious arg exec in the doc/manpage + lore = '' + execer cannot bin/getconf + ''; }; getent = { linux = if stdenv.hostPlatform.libc == "glibc" then pkgs.stdenv.cc.libc.getent @@ -112,6 +118,11 @@ let locale = { linux = pkgs.glibc; darwin = pkgs.darwin.adv_cmds; + # technically just targeting glibc version + # no obvious exec in manpage + lore = '' + execer cannot bin/locale + ''; }; logger = { linux = pkgs.util-linux; @@ -123,6 +134,13 @@ let mount = { linux = pkgs.util-linux; darwin = pkgs.darwin.diskdev_cmds; + # technically just targeting the darwin version; binlore already + # ids the util-linux copy as 'cannot' + # no obvious exec in manpage args; I think binlore flags 'can' + # on the code to run `mount_` variants + lore = '' + execer cannot bin/mount + ''; }; netstat = { linux = pkgs.nettools; @@ -135,6 +153,12 @@ let ps = { linux = pkgs.procps; darwin = pkgs.darwin.ps; + # technically just targeting procps ps (which ids as can) + # but I don't see obvious exec in args; have yet to look + # for underlying cause in source + lore = '' + execer cannot bin/ps + ''; }; quota = { linux = pkgs.linuxquota; @@ -155,6 +179,13 @@ let top = { linux = pkgs.procps; darwin = pkgs.darwin.top; + # technically just targeting procps top; haven't needed this in + # any scripts so far, but overriding it for consistency with ps + # override above and in procps. (procps also overrides 'free', + # but it isn't included here.) + lore = '' + execer cannot bin/top + ''; }; umount = { linux = pkgs.util-linux;