diff --git a/build/deps.sh b/build/deps.sh index 5c80778b24..a66c96ab8b 100755 --- a/build/deps.sh +++ b/build/deps.sh @@ -39,6 +39,7 @@ source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION source deps/from-apt.sh # PY3_BUILD_DEPS #source deps/podman.sh source devtools/run-task.sh # run-task +source test/common.sh # run-task-with-status, nproc # Also in build/dev-shell.sh USER_WEDGE_DIR=~/wedge/oils-for-unix.org @@ -370,19 +371,9 @@ fetch-py() { fetch py_only } -mypy-new() { - local version=0.971 - # Do the latest version for Python 2 - clone-mypy $DEPS_SOURCE_DIR/mypy $version - - local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$version - mkdir -p $dest_dir - - cp --verbose --recursive --no-target-directory \ - $DEPS_SOURCE_DIR/mypy/mypy-$version $dest_dir -} - wedge-exists() { + ### Does an installed wedge already exist? + local is_relative=${3:-} if test -n "$is_relative"; then @@ -426,6 +417,8 @@ download-py3-libs() { } get-typed-ast-patch() { + ### Unused now + curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch } @@ -437,6 +430,8 @@ get-typed-ast-patch() { # # typed_ast is tarred up though patch-typed-ast() { + ### Unused now + local package_dir=_cache/py3-libs local patch=$PWD/deps/typed_ast.patch @@ -522,6 +517,7 @@ install-py3-libs() { $0 install-py3-libs-in-venv $venv_dir $mypy_dir } +# TODO: parallelize this function install-spec-bin() { if ! wedge-exists dash $DASH_VERSION relative; then deps/wedge.sh unboxed-build _build/deps-source/dash @@ -535,14 +531,12 @@ install-spec-bin() { deps/wedge.sh unboxed-build _build/deps-source/busybox fi - #return - - # Compile Error on Fedora - count_all_jobs - # Smoke test error on Alpine + # Fedora compile error - count_all_jobs if ! wedge-exists bash $BASH_VER relative; then deps/wedge.sh unboxed-build _build/deps-source/bash fi + # Fedora compiler error # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken # fallback in Src/Modules/termcap.c that causes a compile error! It seems # like ncurses-devel should fix this, but it doesn't @@ -567,6 +561,63 @@ install-spec-bin() { fi } +py-wedges() { + ### for build/py.sh all + + # Can all be done in parallel + echo cmark $CMARK_VERSION + echo re2c $RE2C_VERSION + echo python2 $PY2_VERSION +} + +cpp-wedges() { + ### for ninja / mycpp translation + + # These are done serially? + echo python3 $PY3_VERSION + echo mypy $MYPY_VERSION + echo py3-libs $PY3_LIBS_VERSION +} + +spec-bin-wedges() { + ### for test/spec-py.sh osh-all + + # Can all be done in parallel + echo bash $BASH_VER + echo dash $DASH_VERSION + echo mksh $MKSH_VERSION + echo zsh $ZSH_VERSION + echo busybox $BUSYBOX_VERSION +} + +maybe-install-wedge() { + local name=$1 + local version=$2 + + if ! wedge-exists $name $version; then + deps/wedge.sh unboxed-build _build/deps-source/$name/ + fi +} + +install-wedges-fast() { + log "" + log "=== Installing $nproc wedges in parallel" + log "" + + # TODO: because we're running in parallel, each one needs to write a separate + # task file. Like the spec tests. + # + # The benchmarks run serially, so they don't do that. + # + # test/wild-runner.sh - dump-html-and-translate-file() + # test/spec-runner.sh - dispatch-one -> run-task-with-status with test-common + + py-wedges | xargs -P $nproc -n 2 -- $0 maybe-install-wedge + + # TODO: collect failures, and exit non-zero if anything fails +} + +# TODO: parallelize this function install-wedges() { local py_only=${1:-} @@ -577,11 +628,11 @@ install-wedges() { # - rel-smoke-test -- mount it in a different location # - Should have a CI task that does all of this! - if ! wedge-exists cmark 0.29.0; then + if ! wedge-exists cmark $CMARK_VERSION; then deps/wedge.sh unboxed-build _build/deps-source/cmark/ fi - if ! wedge-exists re2c 3.0; then + if ! wedge-exists re2c $RE2C_VERSION; then deps/wedge.sh unboxed-build _build/deps-source/re2c/ fi @@ -603,7 +654,6 @@ install-wedges() { $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir fi - # TODO: make the Python build faster by using all your cores? if ! wedge-exists python3 $PY3_VERSION; then deps/wedge.sh unboxed-build _build/deps-source/python3/ fi @@ -613,7 +663,6 @@ install-wedges() { # NOTE: We have to also copy the .git dir, because it has # .git/modules/typeshed - local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION mkdir -p $dest_dir @@ -649,6 +698,10 @@ uftrace-host() { deps/wedge.sh unboxed-build _build/deps-source/uftrace } +install-wedges-py() { + install-wedges py_only +} + R-libs-host() { deps/wedge.sh unboxed-build _build/deps-source/R-libs } @@ -657,10 +710,6 @@ bloaty-host() { deps/wedge.sh unboxed-build _build/deps-source/bloaty } -install-wedges-py() { - install-wedges py_only -} - container-wedges() { ### Build wedges that are copied into containers, not run on host @@ -684,7 +733,6 @@ container-wedges() { # For soil-benchmarks/ images deps/wedge.sh build deps/source.medo/R-libs/ fi - } commas() { @@ -779,7 +827,6 @@ print("Total bytes by file type") for ext, total_bytes in bytes.most_common()[:n]: print("%10d %s" % (total_bytes, ext)) ' | commas - } run-task "$@" diff --git a/core/state.py b/core/state.py index 39545af1fb..e8296bfb62 100644 --- a/core/state.py +++ b/core/state.py @@ -1800,13 +1800,7 @@ def InternalSetGlobal(self, name, new_val): def GetValue(self, name, which_scopes=scope_e.Shopt): # type: (str, scope_t) -> value_t - """Used by the WordEvaluator, ArithEvaluator, ysh/expr_eval.py, etc. - - TODO: - - Many of these should be value.Int, not value.Str - - And even later _pipeline_status etc. should be lists of integers, not - strings - """ + """Used by the WordEvaluator, ArithEvaluator, ExprEvaluator, etc.""" assert isinstance(name, str), name if which_scopes == scope_e.Shopt: @@ -1818,6 +1812,7 @@ def GetValue(self, name, which_scopes=scope_e.Shopt): # if name not in COMPUTED_VARS: ... if name == 'ARGV': + # TODO: ARGV can be a normal mutable variable in YSH items = [value.Str(s) for s in self.GetArgv()] # type: List[value_t] return value.List(items) @@ -1943,7 +1938,7 @@ def GetValue(self, name, which_scopes=scope_e.Shopt): self.line_num.s = str(self.token_for_line.line.line_num) return self.line_num - if name == 'BASHPID': # TODO: Oil name for it + if name == 'BASHPID': # TODO: YSH io->getpid() return value.Str(str(posix.getpid())) if name == '_':