fix #11457 lock down csources version

.builds/freebsd.yml

@@ -21,9 +21,7 @@ tasks:
     sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
       devel/sdl20 devel/sfml www/node devel/gmake devel/git
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources.git
-    gmake -C csources -j $(sysctl -n hw.ncpu)
-    bin/nim c --skipUserCfg --skipParentCfg koch
+    NIMBUILD_ACTION=action_build_koch sh build_all.sh
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim

azure-pipelines.yml

@@ -48,7 +48,7 @@ jobs:
 
     - checkout: self
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
+    - bash: NIMBUILD_ACTION=action_fetch_csources sh build_all.sh
       displayName: 'Checkout csources'
 
     - task: NodeTool@0

build_all.bat

@@ -1,7 +1,13 @@
 @echo off
-rem build development version of the compiler; can be rerun safely
+rem build development version of the compiler; can be rerun safely but won't pickup
+rem modifications in config/build_config.txt after 1st run, see build_all.sh for a way
+
+rem read in some common shared variables (shared with other tools)
+rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
+for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
+
 if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+  git clone -q --depth 1 --branch %nim_csources2_tag% %nim_csources2url% csources
 )
 if not exist bin\nim.exe (
   cd csources
@@ -11,7 +17,7 @@ if not exist bin\nim.exe (
   CALL build.bat
   cd ..
 )
-bin\nim.exe c --skipUserCfg --skipParentCfg koch
-koch.exe boot -d:release --skipUserCfg --skipParentCfg
-koch.exe tools --skipUserCfg --skipParentCfg
+bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg --hints:off
+koch.exe tools --skipUserCfg --skipParentCfg --hints:off 
 

build_all.sh

@@ -1,7 +1,13 @@
 #! /bin/sh
 
-# build development version of the compiler; can be rerun safely.
+# build development version of the compiler; can be rerun safely and will pickup
+# any modification in config/build_config.txt
 # arguments can be passed, eg `--os freebsd`
+#
+# Usage:
+# sh build_all.sh # builds all
+# NIMBUILD_ACTION=action_build_koch sh build_all.sh # just builds everything up to koch
+
 
 set -u # error on undefined variables
 set -e # exit on first error
@@ -11,9 +17,20 @@ echo_run(){
   "$@"
 }
 
-[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
+echo_run . config/build_config.txt
+
+nim_csources=bin/nim_csources2
 
-nim_csources=bin/nim_csources
+fetch_nim_csources(){
+  (
+    [ -d csources ] || echo_run $nim_csources2_clone_cmd
+    echo_run cd csources
+    echo_run git remote set-url origin $nim_csources2url
+    echo_run git fetch -q --depth 1 origin tag $nim_csources2_tag
+    echo_run git checkout $nim_csources2_tag
+    echo_run git reset --hard $nim_csources2_tag
+  )
+}
 
 build_nim_csources_via_script(){
   echo_run cd csources
@@ -23,6 +40,8 @@ build_nim_csources_via_script(){
 build_nim_csources(){
   # avoid changing dir in case of failure
   (
+    [ -f bin/nim ] && echo_run rm bin/nim # otherwise wrongly says: `bin/nim' is up to date.
+    # it's cheap to redo the linking step anyway
     if [ $# -ne 0 ]; then
       # some args were passed (eg: `--cpu i386`), need to call build.sh
       build_nim_csources_via_script "$@"
@@ -37,16 +56,50 @@ build_nim_csources(){
       which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
     fi
   )
+  echo_run bin/nim -v
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
   echo_run cp bin/nim $nim_csources
 }
 
-[ -f $nim_csources ] || echo_run build_nim_csources $@
+## stable API below here
+action_fetch_csources(){
+  echo_run fetch_nim_csources
+}
 
-# Note: if fails, may need to `cd csources && git pull`
-echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+action_build_csources(){
+  action_fetch_csources
+  echo_run build_nim_csources
+}
+
+action_build_koch(){
+  action_build_csources
+  # always bootstrap from $nim_csources for reproducibility, in case this is rerun
+  echo_run $nim_csources c --skipUserCfg --skipParentCfg --hints:off koch
+}
+
+action_build_all(){
+  action_build_koch
+  # re-running without modifications takes 2 seconds up to this line
+  echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
+  echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off # Compile Nimble and other tools.
+}
 
-echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
-echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
+echo "NIMBUILD_ACTION: ${NIMBUILD_ACTION}"
 
+if [ -z "${NIMBUILD_ACTION}" ]; then
+  action_build_all # backward compatibility: same as action_build_all
+elif [ "${NIMBUILD_ACTION}" = "action_definitions" ]; then
+  echo "bash functions defined" # useful if we source this, then we can call individual functions
+elif [ "${NIMBUILD_ACTION}" = "action_fetch_csources" ]; then
+  action_fetch_csources
+elif [ "${NIMBUILD_ACTION}" = "action_build_csources" ]; then
+  echo_run fetch_nim_csources
+elif [ "${NIMBUILD_ACTION}" = "action_build_koch" ]; then
+  action_build_koch
+elif [ "${NIMBUILD_ACTION}" = "action_build_all" ]; then
+  action_build_all
+else
+  echo "unrecognized NIMBUILD_ACTION: $NIMBUILD_ACTION"
+  exit 1
+fi

ci/build.sh

@@ -1,15 +1,11 @@
 sh ci/deps.sh
 
 # Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources.git
-cd csources
-sh build.sh
-cd ..
+NIMBUILD_ACTION=action_build_koch sh build_all.sh
+
 # Add Nim to the PATH
 export PATH=$(pwd)/bin${PATH:+:$PATH}
 # Bootstrap.
-nim -v
-nim c koch
 ./koch boot
 cp bin/nim bin/nimd
 ./koch boot -d:release

config/build_config.txt

@@ -0,0 +1,4 @@
+nim_comment="key-value pairs for windows/posix builds, locking down csources version (refs #11457)"
+nim_csources2url=https://github.com/timotheecour/csources
+nim_csources2_tag=v0.20.0
+nim_csources2_clone_cmd="git clone -q --depth 1 --branch $nim_csources2_tag $nim_csources2url csources"