diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 8d2ccd3..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,35 +0,0 @@ - -name: "Build" -on: - pull_request: - push: - branches: [master, ci] -jobs: - build: - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v2 - with: - # Nix Flakes doesn't work on shallow clones - fetch-depth: 0 - - - uses: cachix/install-nix-action@v12 - with: - install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install - extra_nix_config: | - experimental-features = nix-command flakes - - - uses: cachix/cachix-action@v8 - with: - name: nix-portable - # If you chose API tokens for write access OR if you have a private cache - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - - run: nix build -L . - - - name: Archive result - uses: actions/upload-artifact@v2 - with: - name: nix-portable - path: result/bin/nix-portable diff --git a/.github/workflows/nix-portable.yml b/.github/workflows/nix-portable.yml new file mode 100644 index 0000000..46cbe31 --- /dev/null +++ b/.github/workflows/nix-portable.yml @@ -0,0 +1,118 @@ + +name: "Build" +on: + pull_request: + push: + branches: [master, ci] +jobs: + + + + build: + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + + - uses: cachix/install-nix-action@v12 + with: + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install + extra_nix_config: | + experimental-features = nix-command flakes + + - uses: cachix/cachix-action@v8 + with: + name: nix-portable + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - run: nix build -L . + + - name: Archive result + uses: actions/upload-artifact@v2 + with: + name: nix-portable + path: result/bin/nix-portable + + + + + test_qemu: + name: Test on distro via qemu + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + qemu_os: [ arch, centos7, centos8, debian, ubuntu ] + steps: + + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + + - uses: cachix/install-nix-action@v12 + with: + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install + extra_nix_config: | + experimental-features = nix-command flakes + + - uses: cachix/cachix-action@v8 + with: + name: nix-portable + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - run: nix run -L .#job-qemu-${{ matrix.qemu_os }} + + + + test_docker: + name: Test inside docker container + needs: build + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + + - uses: cachix/install-nix-action@v12 + with: + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install + extra_nix_config: | + experimental-features = nix-command flakes + + - uses: cachix/cachix-action@v8 + with: + name: nix-portable + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - run: nix run -L .#job-docker-debian + + + test_github: + name: Test inside github action + needs: build + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + + - uses: cachix/install-nix-action@v12 + with: + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install + extra_nix_config: | + experimental-features = nix-command flakes + + - uses: cachix/cachix-action@v8 + with: + name: nix-portable + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - run: nix run -L .#job-local diff --git a/.gitignore b/.gitignore index 0c2cee0..1b7fefa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ test/ result* vm/ +.direnv/ +img diff --git a/README.md b/README.md index 23f0eb7..6a98b09 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## Nix Portable Nix - Static, Permissionless, Installation-free, Pre-configured -Nix as a single binary which works without previous installation/configuration and without super user privileges or user namespaces. +Nix as a single binary which doesn't require any configuration, privileges, or namespaces. ### Goals: - make it extremely simple to install nix @@ -9,23 +9,36 @@ Nix as a single binary which works without previous installation/configuration a - be able to use the official binary cache (by simulating the /nix/store) - make it easy to distribute nix (via other package managers) -### Systems confirmed working (Please add yours via PR): - - CentOS 7 - - Debian (in docker) - - NixOS +### Tested on the following systems/environments: + * Distros: + - Arch Linux + - Debian 10 + - CentOS 7 + - CentOS 8 + - NixOS + - Ubuntu 20.04 + * Other Environments: + - Docker (debian image) + - Github Action ### Under the hood: - - the nix-portable binary is a self extracting archive, caching its contents under $HOME/.nix-portable - - either bublewrap (bwrap) or proot is used to simulate the /nix/store directory which actually resides in $HOME/.nix-portable/store - - a default nixpkgs channel is included and the NIX_PATH variable is set accordingly. - - nix version 2.4 is used and configured to enable `flakes` and `nix-command` out of the box. + - The nix-portable binary is a self extracting archive, caching its contents in $HOME/.nix-portable + - Either bublewrap or proot is used to simulate the /nix/store directory which actually resides in $HOME/.nix-portable/store + - A default nixpkgs channel is included and the NIX_PATH variable is set accordingly. + - Nix version 2.4 is used and configured to enable `flakes` and `nix-command` out of the box. + + +### Drawbacks / Considerations: +If user namespaces are not available on a system, nix-portable will fall back to using proot instead of bubblewrap. +Proot's virtualization can have a significant performance overhead depending on the workload. +In that situation, it might be beneficial to use a remote builder or alternatively build the derivations on another host and sync them via a cache like cachix.org. ### Missing Features: - managing nix profiles via `nix-env` - managing nix channels via `nix-channel` - - MacOS - - support other architecutres than x86_64 + - support MacOS + - support other architecutres besides x86_64 ### Executing nix-portable @@ -66,14 +79,14 @@ nix-portable will try to figure out which runtime is best for your system. In case the automatically selected runtime doesn't work, use the follwing environment variables to specify the runtime, but pleaae also open an issue, so we can improve the automatic selection. ### Environmant Variables -The following environment variables are optional and can be used to override the default behaviour of running nix-portable +The following environment variables are optional and can be used to override the default behaviour of nix-portable ``` -NP_DEBUG enable debug logging (to stdout) +NP_DEBUG (1 = debug msgs; 2 = 'set -e' for nix-portable) NP_MINIMAL do not automatically install git NP_LOCATION where to put the `.nix-portable` dir. (defaults to `$HOME`) -NP_RUNTIME which runtime to use (must be either 'bwrap' or 'proot') -NP_BWRAP specify the path to the bwrap executable -NP_PROOT specify the path to the proot executable +NP_RUNTIME which runtime to use (must be 'bwrap' or 'proot') +NP_BWRAP specify the path to the bwrap executable to use +NP_PROOT specify the path to the proot executable to use NP_RUN override the complete command to run nix (to use an unsupported runtime, or for debugging) nix will then be executed like: $NP_RUN {nix-binary} {args...} diff --git a/default.nix b/default.nix index d8cc6a8..989d1c8 100644 --- a/default.nix +++ b/default.nix @@ -57,11 +57,7 @@ let chmod +wx \$dir/bin/${bin}; ''; - installBinBase64 = pkg: bin: '' - (base64 -d> \$dir/bin/${bin} && chmod +x \$dir/bin/${bin}) << END - $(cat ${pkg}/bin/${bin} | base64) - END - ''; + caBundleZstd = pkgs.runCommand "cacerts" {} "cat ${cacert}/etc/ssl/certs/ca-bundle.crt | ${inp.zstd}/bin/zstd -19 > $out"; bwrap = packStaticBin "${inp.bwrap}/bin/bwrap"; proot = packStaticBin "${inp.proot}/bin/proot"; @@ -79,41 +75,74 @@ let #!/usr/bin/env bash set -e + if [ -n "\$NP_DEBUG" ] && [ "\$NP_DEBUG" -ge 2 ]; then + set -x + fi + + if [ -n "\$NP_DEBUG" ]; then + debug(){ + echo \$@ || true + } + else + debug(){ + true + } + fi + # to reference this script's file self="\$(realpath \''${BASH_SOURCE[0]})" - fingerprint="_FINGERPRINT_PLACEHOLDER_" - debug(){ - [ -n "\$NP_DEBUG" ] && echo \$@ || true - } + # fingerprint will be inserted by builder + fingerprint="_FINGERPRINT_PLACEHOLDER_" + # user specified location for program files and nix store [ -z "\$NP_LOCATION" ] && NP_LOCATION="\$HOME" dir="\$NP_LOCATION/.nix-portable" mkdir -p \$dir/bin - - ### setup SSL - # find ssl certs or use from nixpkgs - debug "figuring out ssl certs" - if [ -z "\$SSL_CERT_FILE" ]; then - debug "SSL_CERT_FILE not defined. trying to find certs automatically" - if [ -e /etc/ssl/certs/ca-bundle.crt ]; then - debug "found /etc/ssl/certs/ca-bundle.crt" - export SSL_CERT_FILE=\$(realpath /etc/ssl/certs/ca-bundle.crt) - elif [ ! -e /etc/ssl/certs ]; then - debug "/etc/ssl/certs does not exist, using certs from nixpkgs" - export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt - else - debug "certs seem to reside in /etc/ssl/certs. No need to set up anything" - fi + # the fingerprint being present inside a file indicates that + # this version of nix-portable has already been initialized + if test -e \$dir/conf/fingerprint && [ "\$(cat \$dir/conf/fingerprint)" == "\$fingerprint" ]; then + newNPVersion=false + else + newNPVersion=true fi + # Nix portable ships its own nix.conf + export NIX_CONF_DIR=\$dir/conf/ + + + create_nix_conf(){ + sandbox=\$1 + + mkdir -p \$dir/conf/ + rm -f \$dir/conf/nix.conf + + echo "build-users-group = " > \$dir/conf/nix.conf + echo "experimental-features = nix-command flakes" >> \$dir/conf/nix.conf + echo "use-sqlite-wal = false" >> \$dir/conf/nix.conf + echo "sandbox-paths = /bin/sh=\$dir/busybox/bin/busybox" >> \$dir/conf/nix.conf + + echo "sandbox = \$sandbox" >> \$dir/conf/nix.conf + } + + + ### install files + + PATH_OLD="\$PATH" + + # as soon as busybox is unpacked, restrict PATH to busybox to ensure reproducibility of this script + # only unpack binaries if necessary + if [ "\$newNPVersion" == "false" ]; then - ### install binaries - if test -e \$dir/fingerprint && [ "\$(cat \$dir/fingerprint)" == "\$fingerprint" ]; then debug "binaries already installed" + export PATH="\$dir/busybox/bin" + else - debug "installing binaries" + + debug "installing files" + + mkdir -p \$dir/emptyroot # install busybox mkdir -p \$dir/busybox/bin @@ -125,32 +154,92 @@ let [ ! -e "\$dir/busybox/bin/\$bin" ] && ln -s busybox "\$dir/busybox/bin/\$bin" done + export PATH="\$dir/busybox/bin" + # install other binaries - ${installBinBase64 zstd "zstd"} + ${installBin zstd "zstd"} ${installBin proot "proot"} ${installBin bwrap "bwrap"} - ${installBin zstd "zstd"} - # save fingerprint - echo -n "\$fingerprint" > "\$dir/fingerprint" + # install ssl cert bundle + unzip -poj "\$self" ${ lib.removePrefix "/" "${caBundleZstd}"} | \$dir/bin/zstd -d > \$dir/ca-bundle.crt + + create_nix_conf false + fi - ### add busybox to PATH - export PATH="\$PATH:\$dir/busybox/bin" + + ### setup SSL + # find ssl certs or use from nixpkgs + debug "figuring out ssl certs" + if [ -z "\$SSL_CERT_FILE" ]; then + debug "SSL_CERT_FILE not defined. trying to find certs automatically" + if [ -e /etc/ssl/certs/ca-bundle.crt ]; then + export SSL_CERT_FILE=\$(realpath /etc/ssl/certs/ca-bundle.crt) + debug "found /etc/ssl/certs/ca-bundle.crt which is a symlink to \$SSL_CERT_FILE" + elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then + export SSL_CERT_FILE=\$(realpath /etc/ssl/certs/ca-certificates.crt) + debug "found /etc/ssl/certs/ca-certificates.crt which is a symlink to \$SSL_CERT_FILE" + elif [ ! -e /etc/ssl/certs ]; then + debug "/etc/ssl/certs does not exist. Will use certs from nixpkgs." + export SSL_CERT_FILE=\$dir/ca-bundle.crt + else + debug "certs seem to reside in /etc/ssl/certs. No need to set up anything" + fi + fi + if [ -n "\$SSL_CERT_FILE" ]; then + sslBind="\$SSL_CERT_FILE" + else + sslBind=/etc/ssl + fi + - ### gather paths to bind for proot - paths="\$(find / -mindepth 1 -maxdepth 1 -not -name etc)" - paths="\$paths /etc/host.conf /etc/hosts /etc/hosts.equiv /etc/mtab /etc/netgroup /etc/networks /etc/passwd /etc/group /etc/nsswitch.conf /etc/resolv.conf /etc/localtime $HOME" - toBind="" - mkdir -p \$dir/shared-files - for p in \$paths; do - if [ -e "\$p" ]; then - real=\$(realpath \$p) - [ -e "\$real" ] && toBind="\$toBind \$real \$p" + ### detecting existing git installation + if [ -n "\$NP_MINIMAL" ]; then + doInstallGit=false + else + if ! (PATH="\$PATH_OLD:\$PATH" which git &>/dev/null) ; then + doInstallGit=true + else + doInstallGit=false + # bind the old git to \$NP_LOCATION/.nix-portable/git/bin + gitBind="\$(dirname \$(realpath \$(PATH="\$PATH_OLD:\$PATH" which git))) \$NP_LOCATION/.nix-portable/git/bin" fi - done + fi + + + + collectProotBinds(){ + ### gather paths to bind for proot + # we cannot bind / to / without running into a lot of trouble, therefore + # we need to collect all top level directories and bind them inside an empty root + paths="\$(find / -mindepth 1 -maxdepth 1 -not -name etc)" + paths="\$paths /etc/host.conf /etc/hosts /etc/hosts.equiv /etc/mtab /etc/netgroup /etc/networks /etc/passwd /etc/group /etc/nsswitch.conf /etc/resolv.conf /etc/localtime $HOME" + + toBind="" + for p in \$paths; do + if [ -e "\$p" ]; then + real=\$(realpath \$p) + [ -e "\$real" ] && toBind="\$toBind \$real \$p" + fi + done + + toBind="\$toBind \$gitBind" + } + + collectBwrapBinds(){ + toBind="" + # if we're on a nixos, the /bin/sh symlink will point + # to a /nix/store path which doesn't exit inside the wrapped env + # we fix this by binding busybox/bin to /bin + if test -s /bin/sh && [[ "\$(realpath /bin/sh)" == /nix/store/* ]]; then + toBind="\$toBind \$dir/busybox/bin /bin" + fi + + toBind="\$toBind \$gitBind" + } makeBindArgs(){ arg=\$1; shift @@ -168,17 +257,18 @@ let } + ### select container runtime debug "figuring out which runtime to use" - [ -z "\$NP_BWRAP" ] && NP_BWRAP=\$(which bwrap 2>/dev/null) || true + [ -z "\$NP_BWRAP" ] && NP_BWRAP=\$(PATH="\$PATH_OLD:\$PATH" which bwrap 2>/dev/null) || true [ -z "\$NP_BWRAP" ] && NP_BWRAP=\$dir/bin/bwrap debug "bwrap executable: \$NP_BWRAP" - [ -z "\$NP_PROOT" ] && NP_PROOT=\$(which proot 2>/dev/null) || true + [ -z "\$NP_PROOT" ] && NP_PROOT=\$(PATH="\$PATH_OLD:\$PATH" which proot 2>/dev/null) || true [ -z "\$NP_PROOT" ] && NP_PROOT=\$dir/bin/proot debug "proot executable: \$NP_PROOT" if [ -z "\$NP_RUNTIME" ]; then # check if bwrap works properly - if \$NP_BWRAP --bind / / --bind \$dir/busybox/bin/busybox "\$HOME/.nix-portable/true" "\$HOME/.nix-portable/true" 2>/dev/null ; then + if \$NP_BWRAP --bind / / --bind \$dir/ /nix --bind \$dir/busybox/bin/busybox "\$HOME/.nix-portable/true" "\$HOME/.nix-portable/true" 2>/dev/null ; then debug "bwrap seems to work on this system -> will use bwrap" NP_RUNTIME=bwrap else @@ -188,44 +278,30 @@ let else debug "runtime selected via NP_RUNTIME : \$NP_RUNTIME" fi - mkdir -p \$dir/emptyroot if [ "\$NP_RUNTIME" == "bwrap" ]; then - # makeBindArgs --bind " " \$toBind - if [ -n "\$SSL_CERT_FILE" ]; then - makeBindArgs --bind " " \$SSL_CERT_FILE \$SSL_CERT_FILE - fi + collectBwrapBinds + makeBindArgs --bind " " \$toBind \$sslBind \$sslBind run="\$NP_BWRAP \$BWRAP_ARGS \\ --bind / /\\ --dev-bind /dev /dev\\ --bind \$dir/ /nix\\ \$binds" + # --bind \$dir/busybox/bin/busybox /bin/sh\\ else + # proot + collectProotBinds makeBindArgs -b ":" \$toBind - binds_1="\$binds" - if [ -n "\$SSL_CERT_FILE" ]; then - debug "creating bind args for \$SSL_CERT_FILE" - makeBindArgs -b ":" \$SSL_CERT_FILE \$SSL_CERT_FILE - else - debug "creating bind args for /etc/ssl" - makeBindArgs -b ":" /etc/ssl /etc/ssl - fi - binds="\$binds_1 \$binds" + bindsAll="\$binds" + makeBindArgs -b ":" \$sslBind \$sslBind + binds="\$bindsAll \$binds" run="\$NP_PROOT \$PROOT_ARGS\\ - -R \$dir/emptyroot - -b \$dir/store:/nix/store + -R \$dir/emptyroot\\ + -b \$dir/store:/nix/store\\ \$binds" + # -b \$dir/busybox/bin/busybox:/bin/sh\\ fi - ### generate nix config - mkdir -p \$dir/conf/ - echo "build-users-group = " > \$dir/conf/nix.conf - echo "experimental-features = nix-command flakes" >> \$dir/conf/nix.conf - echo "sandbox = true" >> \$dir/conf/nix.conf - echo "sandbox-fallback = true" >> \$dir/conf/nix.conf - echo "use-sqlite-wal = false" >> \$dir/conf/nix.conf - export NIX_CONF_DIR=\$dir/conf/ - ### setup environment export NIX_PATH="\$dir/channels:nixpkgs=\$dir/channels/nixpkgs" @@ -237,8 +313,6 @@ let # Install all the nix store paths necessary for the current nix-portable version # We only unpack missing store paths from the tar archive. # xz must be in PATH - PATH_OLD="\$PATH" - PATH="\$dir/bin/:\$PATH" index="$(cat ${storeTar}/index)" export missing=\$( @@ -250,20 +324,21 @@ let ) if [ -n "\$missing" ]; then + debug "extracting missing store paths" ( mkdir -p \$dir/tmp \$dir/store/ rm -rf \$dir/tmp/* cd \$dir/tmp unzip -qqp "\$self" ${ lib.removePrefix "/" "${storeTar}/tar"} \ - | tar -x --zstd \$missing --strip-components 2 + | \$dir/bin/zstd -d \ + | tar -x \$missing --strip-components 2 mv \$dir/tmp/* \$dir/store/ ) + rm -rf \$dir/tmp fi - PATH="\$PATH_OLD" - if [ -n "\$missing" ]; then - debug "loading new store paths" + debug "registering new store paths to DB" reg="$(cat ${storeTar}/closureInfo/registration)" cmd="\$run \$dir/store${lib.removePrefix "/nix/store" nix}/bin/nix-store --load-db" debug "running command: \$cmd" @@ -271,22 +346,6 @@ let fi - ### install git via nix, if git not installed yet or git installation is in /nix path - if [ -z "\$NP_MINIMAL" ] && ( ! which git &>/dev/null || [[ "\$(realpath \$(which git))" == /nix/* ]] ); then - needGit=true - else - needGit=false - fi - debug "needGit: \$needGit" - if \$needGit && [ ! -e \$dir/store${lib.removePrefix "/nix/store" git.out} ] ; then - echo "Installing git. Disable this by setting 'NP_MINIMAL=1'" - \$run \$dir/store${lib.removePrefix "/nix/store" nix}/bin/nix build --impure --no-link --expr " - (import ${nixpkgsSrc} {}).git.out - " - else - debug "git already installed or not required" - fi - ### select executable # the executable can either be selected by executing './nix-portable BIN_NAME', @@ -308,20 +367,74 @@ let fi + + ### check which runtime has been used previously + lastRuntime=\$(cat "\$dir/conf/last_runtime" 2>/dev/null) || true + + + + ### check if nix is funtional with or without sandbox + # sandbox-fallback is not reliable: https://github.com/NixOS/nix/issues/4719 + if [ "\$newNPVersion" == "true" ] || [ "\$lastRuntime" != "\$NP_RUNTIME" ]; then + nixBin="\$dir/store${lib.removePrefix "/nix/store" nix}/bin/nix-build" + debug "Testing if nix can build stuff without sandbox" + if ! \$run "\$nixBin" -E "(import {}).runCommand \\"test\\" {} \\"echo \$(date) > \\\$out\\"" --option sandbox false &>/dev/null; then + echo "Fatal error: nix is unable to build packages" + exit 1 + fi + + debug "Testing if nix sandox is functional" + if ! \$run "\$nixBin" -E "(import {}).runCommand \\"test\\" {} \\"echo \$(date) > \\\$out\\"" --option sandbox true &>/dev/null; then + debug "Sandbox doesn't work -> disabling sandbox" + create_nix_conf false + else + debug "Sandboxed builds work -> enabling sandbox" + create_nix_conf true + fi + + fi + + + ### save fingerprint and lastRuntime + if [ "\$newNPVersion" == "true" ]; then + echo -n "\$fingerprint" > "\$dir/conf/fingerprint" + fi + if [ "\$lastRuntime" != \$NP_RUNTIME ]; then + echo -n \$NP_RUNTIME > "\$dir/conf/last_runtime" + fi + + + ### set PATH - # add git - \$needGit && export PATH="\$PATH:${git.out}/bin" + # restore original PATH and append busybox + export PATH="\$PATH_OLD:\$dir/busybox/bin" + + + + ### install git via nix, if git not installed or git installation is in /nix path + if \$doInstallGit && [ ! -e \$dir/store${lib.removePrefix "/nix/store" git.out} ] ; then + echo "Installing git because it's missing. Disable this by setting 'NP_MINIMAL=1'" + \$run \$dir/store${lib.removePrefix "/nix/store" nix}/bin/nix build --impure --no-link --expr " + (import ${nixpkgsSrc} {}).git.out + " + else + debug "git already installed or not required" + fi + + # add git if necessary + \$doInstallGit && export PATH="\$PATH:${git.out}/bin" + ### run commands [ -z "\$NP_RUN" ] && NP_RUN="\$run" if [ "\$NP_RUNTIME" == "proot" ]; then debug "running command: \$NP_RUN \$bin \$@" - exec \$NP_RUN \$bin "\$@" + exec \$NP_RUN \$bin "\$@" else cmd="\$NP_RUN \$bin \$@" debug "running command: \$cmd" - exec \$cmd + exec \$NP_RUN \$bin "\$@" fi ''; @@ -351,6 +464,7 @@ let $zip $out/bin/nix-portable.zip ${bwrap}/bin/bwrap $zip $out/bin/nix-portable.zip ${zstd}/bin/zstd $zip $out/bin/nix-portable.zip ${storeTar}/tar + $zip $out/bin/nix-portable.zip ${caBundleZstd} # create fingerprint fp=$(sha256sum $out/bin/nix-portable.zip | cut -d " " -f 1) diff --git a/flake.nix b/flake.nix index 6a29e08..5002cde 100644 --- a/flake.nix +++ b/flake.nix @@ -7,9 +7,47 @@ flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { self, ... }@inp: + outputs = { self, ... }@inp: + with builtins; with inp.nixpkgs.lib; let + + # Linux distro images to test nix-portable against + # After adding a new system, don't forget to add the name also in ./.github/workflows + testImages = { + arch = { + url = "https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/20342/artifacts/raw/output/Arch-Linux-x86_64-basic-20210420.20342.qcow2"; + sha256 = "b59f7218df206b135a0cd9a288e79e35cf892bca0c71373588d0d10b029d50a4"; + extraVirtCustomizeCommands = [ + "--run-command 'systemctl disable pacman-init'" + "--run-command 'systemctl disable reflector-init'" + ]; + }; + centos7 = { + url = "https://cloud.centos.org/altarch/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2c"; + sha256 = "09wqzlhb858qm548ak4jj4adchxn7rgf5fq778hrc52rjqym393v"; + }; + centos8 = { + url = "https://cloud.centos.org/altarch/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2"; + sha256 = "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0"; + }; + debian = { + url = "https://cdimage.debian.org/cdimage/openstack/archive/10.9.0/debian-10.9.0-openstack-amd64.qcow2"; + sha256 = "0mf9k3pgzighibly1sy3cjq7c761r3akp8mlgd878lwf006vqrky"; + }; + ubuntu = { + url = "https://cloud-images.ubuntu.com/focal/20210415/focal-server-cloudimg-amd64.img"; + sha256 = "38b82727bfc1b36d9784bf07b8368c1d777450e978837e1cd7fa32b31837e77c"; + extraVirtCustomizeCommands = [ + "--copy-in ${./testing/ubuntu}/01-netplan.yaml:/etc/netplan/" + ]; + }; + }; + + commandsToTest = [ + "nix build -L --impure --expr '(import {}).hello.overrideAttrs(_:{change=1;})'" + "nix-shell -p hello --run hello" + ]; nixPortableForSystem = { system, crossSystem ? null, }: let @@ -42,12 +80,122 @@ xz = pkgs.pkgsStatic.xz; zstd = pkgs.pkgsStatic.zstd; }; + + prepareCloudImage = pkgs: qcowImg: pkgs.runCommand "img-with-ssh" {} '' + ${pkgs.libguestfs-with-appliance}/virt-sysprep --version exit 1 + ''; in recursiveUpdate - (inp.flake-utils.lib.eachDefaultSystem (system: rec { + (inp.flake-utils.lib.eachDefaultSystem (system: let pkgs = inp.nixpkgs.legacyPackages."${system}"; in rec { + devShell = pkgs.mkShell { + buildInputs = with pkgs; [ + libguestfs-with-appliance + qemu + bashInteractive + ]; + }; packages.nix-portable = nixPortableForSystem { inherit system; }; defaultPackage = packages.nix-portable; + apps = + let + makeQemuPipelines = debug: mapAttrs' (os: img: + nameValuePair + "job-qemu-${os}${optionalString debug "-debug"}" + { + type = "app"; + program = toString (pkgs.writeScript "job-qemu-${os}" '' + #!/usr/bin/env bash + set -e + + img=${fetchurl { inherit (testImages."${os}") url sha256 ;}} + pubKey=${./testing}/id_ed25519.pub + privKey=${./testing}/id_ed25519 + nixPortable=${packages.nix-portable}/bin/nix-portable + ssh="${pkgs.openssh}/bin/ssh -p 10022 -i $privKey -o StrictHostKeyChecking=no test@localhost" + sshRoot="${pkgs.openssh}/bin/ssh -p 10022 -i $privKey -o StrictHostKeyChecking=no root@localhost" + + setup_and_start_vm() { + cat $img > /tmp/img + + ${pkgs.libguestfs-with-appliance}/bin/virt-customize -a /tmp/img \ + --run-command 'useradd test && mkdir -p /home/test && chown test.test /home/test' \ + --run-command 'ssh-keygen -A' \ + --ssh-inject test:file:$pubKey \ + --ssh-inject root:file:$pubKey \ + --copy-in $nixPortable:/home/test/ \ + ${concatStringsSep " " (testImages."${os}".extraVirtCustomizeCommands or [])} \ + ${optionalString debug "--root-password file:${pkgs.writeText "pw" "root"}"} \ + --selinux-relabel + + ${pkgs.qemu}/bin/qemu-system-x86_64 \ + -hda /tmp/img \ + -m 2048 \ + -netdev user,hostfwd=tcp::10022-:22,id=n1 \ + -device virtio-net-pci,netdev=n1 \ + ${optionalString (! debug) "-nographic"} \ + & + } + + # if debug, dont init/run VM if already running + ${optionalString debug '' + ${pkgs.busybox}/bin/pgrep qemu >/dev/null || \\ + ''} + setup_and_start_vm + + while ! $ssh -o ConnectTimeout=2 true 2>/dev/null ; do + echo "waiting for ssh" + sleep 1 + done + + ${optionalString debug '' + $sshRoot "rm -rf /home/test/.nix-portable /home/test/nix-portable" + scp -P 10022 -i $privKey -o StrictHostKeyChecking=no ${packages.nix-portable}/bin/nix-portable test@localhost:/home/test/nix-portable + ''} + + echo -e "\n\nstarting to test nix-portable" + + # test some nix commands + ${concatStringsSep "\n" (map (cmd: + ''$ssh "NP_DEBUG=1 NP_MINIMAL=1 /home/test/nix-portable ${cmd}"'' + ) commandsToTest)} + + echo "all tests succeeded" + ''); + } + ) testImages; + in + # generate jobs with and without debug settings + makeQemuPipelines true // makeQemuPipelines false + # add + // { + job-docker-debian.type = "app"; + job-docker-debian.program = toString (pkgs.writeScript "job-docker-debian" '' + #!/usr/bin/env bash + set -e + DOCKER_CMD="''${DOCKER_CMD:-docker}" + baseCmd="\ + $DOCKER_CMD run -i --rm \ + -v ${packages.nix-portable}/bin/nix-portable:/nix-portable \ + -e NP_MINIMAL=1 \ + -e NP_DEBUG=1" + if [ -n "$1" ]; then + $baseCmd -it debian $1 + else + ${concatStringsSep "\n" (map (cmd: "$baseCmd debian /nix-portable ${cmd}") commandsToTest)} + fi + ''); + job-local.type = "app"; + job-local.program = toString (pkgs.writeScript "job-local" '' + #!/usr/bin/env bash + set -e + export NP_DEBUG=1 + export NP_MINIMAL=1 + ${concatStringsSep "\n" (map (cmd: + ''${packages.nix-portable}/bin/nix-portable ${cmd}'' + ) commandsToTest)} + ''); + }; })) { packages = (genAttrs [ "x86_64-linux" ] (system: (listToAttrs (map (crossSystem: diff --git a/testing/id_ed25519 b/testing/id_ed25519 new file mode 100644 index 0000000..6b3a0e5 --- /dev/null +++ b/testing/id_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACCkGe7gNv3mTzONb+Dsf0YmlCdyTNauDe5QcoHh6tCEhwAAAJg2B2NyNgdj +cgAAAAtzc2gtZWQyNTUxOQAAACCkGe7gNv3mTzONb+Dsf0YmlCdyTNauDe5QcoHh6tCEhw +AAAEBAQoC1JvAToiYi5dY+3r3YNVq7CMOVXWpecPCqmgPiNaQZ7uA2/eZPM41v4Ox/RiaU +J3JM1q4N7lBygeHq0ISHAAAAD2dybXBmQGdybXBmLW5peAECAwQFBg== +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/id_ed25519.pub b/testing/id_ed25519.pub new file mode 100644 index 0000000..a7da915 --- /dev/null +++ b/testing/id_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKQZ7uA2/eZPM41v4Ox/RiaUJ3JM1q4N7lBygeHq0ISH grmpf@grmpf-nix diff --git a/testing/ubuntu/01-netplan.yaml b/testing/ubuntu/01-netplan.yaml new file mode 100644 index 0000000..e5bdd37 --- /dev/null +++ b/testing/ubuntu/01-netplan.yaml @@ -0,0 +1,6 @@ +network: + version: 2 + renderer: networkd + ethernets: + ens3: + dhcp4: yes