binlore: migrate override lore to package passthru

Lore overrides have been included with binlore's source up to now, but
this hasn't worked very well. (It isn't as easy to self-service for
people working in nixpkgs, and its use of partial pnames for matching
breaks down around some edge cases like version numbers appearing
early in perl pnames, or multiple packages having identical pnames.)

pkgs/by-name/pd/pdf2odt/package.nix

@@ -49,6 +49,10 @@ resholve.mkDerivation rec {
       imagemagick
       zip
     ];
+    execer = [
+      # zip can exec; confirmed 2 invocations in pdf2odt don't
+      "cannot:${zip}/bin/zip"
+    ];
   };
 
   meta = with lib; {

pkgs/development/libraries/libarchive/default.nix

@@ -23,6 +23,9 @@
 , cmake
 , nix
 , samba
+
+# for passthru.lore
+, binlore
 }:
 
 assert xarSupport -> libxml2 != null;
@@ -124,4 +127,11 @@ stdenv.mkDerivation (finalAttrs: {
   passthru.tests = {
     inherit cmake nix samba;
   };
+
+  # bsdtar is detected as "cannot" because its exec is internal to
+  # calls it makes into libarchive itself. If binlore gains support
+  # for detecting another layer down into libraries, this can be cut.
+  passthru.lore = (binlore.synthesize finalAttrs.finalPackage ''
+    execer can bin/bsdtar
+  '');
 })

pkgs/development/libraries/ncurses/default.nix

@@ -10,6 +10,7 @@
 , mouseSupport ? false, gpm
 , unicodeSupport ? true
 , testers
+, binlore
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -169,6 +170,17 @@ stdenv.mkDerivation (finalAttrs: {
     rm "$out"/lib/*.a
   '';
 
+  # I'm not very familiar with ncurses, but it looks like most of the
+  # exec here will run hard-coded executables. There's one that is
+  # dynamic, but it looks like it only comes from executing a terminfo
+  # file, so I think it isn't going to be under user control via CLI?
+  # Happy to have someone help nail this down in either direction!
+  # The "capability" is 'iprog', and I could only find 1 real example:
+  # https://invisible-island.net/ncurses/terminfo.ti.html#tic-linux-s
+  passthru.lore = (binlore.synthesize ncurses ''
+    execer cannot bin/{reset,tput,tset}
+  '');
+
   meta = with lib; {
     homepage = "https://www.gnu.org/software/ncurses/";
     description = "Free software emulation of curses in SVR4 and more";

pkgs/development/misc/resholve/test.nix

@@ -25,6 +25,39 @@
 , rlwrap
 , gnutar
 , bc
+# override testing
+, esh
+, getconf
+, libarchive
+, locale
+, mount
+, ncurses
+, nixos-install-tools
+, nixos-rebuild
+, procps
+, ps
+# known consumers
+, aaxtomp3
+, arch-install-scripts
+, bashup-events32
+, dgoss
+, git-ftp
+, ix
+, lesspipe
+, locate-dominating-file
+, mons
+, msmtp
+, nix-direnv
+, pdf2odt
+, pdfmm
+, rancid
+, s0ix-selftest-tool
+, unix-privesc-check
+, wgnord
+, wsl-vpnkit
+, xdg-utils
+, yadm
+, zxfer
 }:
 
 let
@@ -190,4 +223,71 @@ rec {
     echo "Hello"
     file .
   '';
+  # spot-check lore overrides
+  loreOverrides = resholve.writeScriptBin "verify-overrides" {
+    inputs = [
+      coreutils
+      esh
+      getconf
+      libarchive
+      locale
+      mount
+      ncurses
+      procps
+      ps
+    ] ++ lib.optionals stdenv.isLinux [
+      nixos-install-tools
+      nixos-rebuild
+    ];
+    interpreter = "none";
+    execer = [
+      "cannot:${esh}/bin/esh"
+    ];
+    fix = {
+      mount = true;
+    };
+  } (''
+    env b2sum fake args
+    b2sum fake args
+    esh fake args
+    getconf fake args
+    bsdtar fake args
+    locale fake args
+    mount fake args
+    reset fake args
+    tput fake args
+    tset fake args
+    ps fake args
+    top fake args
+  '' + lib.optionalString stdenv.isLinux ''
+    nixos-generate-config fake args
+    nixos-rebuild fake args
+  '');
+
+  # ensure known consumers in nixpkgs keep working
+  inherit aaxtomp3;
+  inherit bashup-events32;
+  inherit bats;
+  inherit git-ftp;
+  inherit ix;
+  inherit lesspipe;
+  inherit locate-dominating-file;
+  inherit mons;
+  inherit msmtp;
+  inherit nix-direnv;
+  inherit pdf2odt;
+  inherit pdfmm;
+  inherit shunit2;
+  inherit xdg-utils;
+  inherit yadm;
+} // lib.optionalAttrs stdenv.isLinux {
+  inherit arch-install-scripts;
+  inherit dgoss;
+  inherit rancid;
+  inherit unix-privesc-check;
+  inherit wgnord;
+  inherit wsl-vpnkit;
+  inherit zxfer;
+} // lib.optionalAttrs (stdenv.isLinux && (stdenv.isi686 || stdenv.isx86_64)) {
+  inherit s0ix-selftest-tool;
 }

pkgs/development/tools/analysis/binlore/default.nix

@@ -55,58 +55,156 @@ let
     #   in here, but I'm erring on the side of flexibility
     #   since this form will make it easier to pilot other
     #   uses of binlore.
-    callback = lore: drv: overrides: ''
+    callback = lore: drv: ''
       if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then
         echo generating binlore for $drv by running:
         echo "${pkgsBuildBuild.yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${pkgsBuildBuild.yallback}/bin/yallback ${lore.yallback}"
       else
         echo "failed to generate binlore for $drv (none of ${drv}/{bin,lib,libexec} exist)"
       fi
-    '' +
-    /*
-    Override lore for some packages. Unsure, but for now:
-    1. start with the ~name (pname-version)
-    2. remove characters from the end until we find a match
-       in overrides/
-    3. execute the override script with the list of expected
-       lore types
-    */
-    ''
-      i=''${#identifier}
-      filter=
-      while [[ $i > 0 ]] && [[ -z "$filter" ]]; do
-        if [[ -f "${overrides}/''${identifier:0:$i}" ]]; then
-          filter="${overrides}/''${identifier:0:$i}"
-          echo using "${overrides}/''${identifier:0:$i}" to generate overriden binlore for $drv
-          break
-        fi
-        ((i--)) || true # don't break build
-      done # || true # don't break build
+
       if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then
-        ${pkgsBuildBuild.yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${pkgsBuildBuild.yallback}/bin/yallback ${lore.yallback} "$filter"
+        ${pkgsBuildBuild.yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${pkgsBuildBuild.yallback}/bin/yallback ${lore.yallback}
       fi
     '';
   };
-  overrides = (src + "/overrides");
 
 in rec {
+  /*
+    Output a directory containing lore for multiple drvs.
+
+    This will `make` lore for drv in drvs and then combine lore
+    of the same type across all packages into a single file.
+
+    When drvs are also specified in the strip argument, corresponding
+    lore is made relative by stripping the path of each drv from
+    matching entries. (This is mainly useful in a build process that
+    uses a chain of two or more derivations where the output of one
+    is the source for the next. See resholve for an example.)
+  */
   collect = { lore ? loreDef, drvs, strip ? [ ] }: (runCommand "more-binlore" { } ''
     mkdir $out
     for lorefile in ${toString lore.types}; do
       cat ${lib.concatMapStrings (x: x + "/$lorefile ") (map (make lore) (map lib.getBin (builtins.filter lib.isDerivation drvs)))} > $out/$lorefile
-      substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace '${x}/' '' ") strip}
+      substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace-quiet '${x}/' '' ") strip}
     done
   '');
-  # TODO: echo for debug, can be removed at some point
+
+  /*
+    Output a directory containing lore for a single drv.
+
+    This produces lore for the derivation (via lore.callback) and
+    appends any lore that the derivation itself wrote to nix-support
+    or which was overridden in drv.lore (passthru).
+
+    Since the last entry wins, the effective priority is:
+    drv.lore > $drv/nix-support > lore generated here by callback
+  */
   make = lore: drv: runCommand "${drv.name}-binlore" {
-      identifier = drv.name;
       drv = drv;
     } (''
     mkdir $out
     touch $out/{${builtins.concatStringsSep "," lore.types}}
 
-    ${lore.callback lore drv overrides}
+    ${lore.callback lore drv}
+    '' +
+    # append lore from package's $out and drv.lore (last entry wins)
+    ''
+    for lore_type in ${builtins.toString lore.types}; do
+      if [[ -f "${drv}/nix-support/$lore_type" ]]; then
+        cat "${drv}/nix-support/$lore_type" >> "$out/$lore_type"
+      fi
+      '' + lib.optionalString (builtins.hasAttr "lore" drv) ''
+      if [[ -f "${drv.lore}/$lore_type" ]]; then
+        cat "${drv.lore}/$lore_type" >> "$out/$lore_type"
+      fi
+      '' + ''
+    done
 
     echo binlore for $drv written to $out
   '');
+
+  /*
+    Utility function for creating override lore for drv.
+
+    In the normal case, we attach this lore to `drv.passthru.lore`.
+
+    The lore argument should be a Shell script (string) that generates
+    the necessary lore. You can use arbitrary Shell, but this function
+    includes a shell DSL you can use to declare/generate lore in most
+    cases. It has the following functions:
+
+    - `execer <verdict> [<path>...]`
+    - `wrapper <wrapper_path> <original_path>`
+
+    Writing every override explicitly in a Nix list would be tedious
+    for large packages, but this small shell DSL enables us to express
+    many overrides efficiently via pathname expansion/globbing.
+
+    Here's a very general example of both functions:
+
+    passthru.lore = (binlore.synthesize finalAttrs.finalPackage ''
+      execer can bin/hello bin/{a,b,c}
+      wrapper bin/hello bin/.hello-wrapped
+    '');
+
+    And here's a specific example of how pathname expansion enables us
+    to express lore for the single-binary variant of coreutils while
+    being both explicit and (somewhat) efficient:
+
+    passthru = {} // optionalAttrs (singleBinary != false) {
+      lore = (binlore.synthesize coreutils ''
+        execer can bin/{chroot,env,install,nice,nohup,runcon,sort,split,stdbuf,timeout}
+        execer cannot bin/{[,b2sum,base32,base64,basename,basenc,cat,chcon,chgrp,chmod,chown,cksum,comm,cp,csplit,cut,date,dd,df,dir,dircolors,dirname,du,echo,expand,expr,factor,false,fmt,fold,groups,head,hostid,id,join,kill,link,ln,logname,ls,md5sum,mkdir,mkfifo,mknod,mktemp,mv,nl,nproc,numfmt,od,paste,pathchk,pinky,pr,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,shred,shuf,sleep,stat,stty,sum,sync,tac,tail,tee,test,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,vdir,wc,who,whoami,yes}
+      '');
+    };
+
+    Caution: Be thoughtful about using a bare wildcard (*) glob here.
+    We should generally override lore only when a human understands if
+    the executable will exec arbitrary user-passed executables. A bare
+    glob can match new executables added in future package versions
+    before anyone can audit them.
+  */
+  synthesize = drv: lore: runCommand "${drv.name}-lore-override" {
+    drv = drv;
+  } (''
+    execer(){
+      local verdict="$1"
+
+      shift
+
+      for path in "$@"; do
+        if [[ -e "$PWD/$path" ]]; then
+          echo "$verdict:$PWD/$path"
+        else
+          echo "error: Tried to synthesize execer lore for missing file: $PWD/$path" >&2
+          exit 2
+        fi
+      done
+    } >> $out/execers
+
+    wrapper(){
+      local wrapper="$1"
+      local original="$2"
+
+      if [[ ! -e "$wrapper" ]]; then
+        echo "error: Tried to synthesize wrapper lore for missing wrapper: $PWD/$wrapper" >&2
+        exit 2
+      fi
+
+      if [[ ! -e "$original" ]]; then
+        echo "error: Tried to synthesize wrapper lore for missing original: $PWD/$original" >&2
+        exit 2
+      fi
+
+      echo "$PWD/$wrapper:$PWD/$original"
+
+    } >> $out/wrappers
+
+    mkdir $out
+
+    # lore override commands are relative to the drv root
+    cd $drv
+
+  '' + lore);
 }

pkgs/os-specific/linux/nixos-rebuild/default.nix

@@ -10,6 +10,8 @@
 , lib
 , nixosTests
 , installShellFiles
+, binlore
+, nixos-rebuild
 }:
 let
   fallback = import ./../../../../nixos/modules/installer/tools/nix-fallback-paths.nix;
@@ -49,6 +51,13 @@ substitute {
     target-host = nixosTests.nixos-rebuild-target-host;
   };
 
+  # nixos-rebuild can’t execute its arguments
+  # (but it can run ssh with the with the options stored in $NIX_SSHOPTS,
+  # and ssh can execute its arguments...)
+  passthru.lore = (binlore.synthesize nixos-rebuild ''
+    execer cannot bin/nixos-rebuild
+  '');
+
   meta = {
     description = "Rebuild your NixOS configuration and switch to it, on local hosts and remote.";
     homepage = "https://github.com/NixOS/nixpkgs/tree/master/pkgs/os-specific/linux/nixos-rebuild";