From 6047ad167739f51e8cb3ffc04d8bbe4c8de9d6da Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 17:43:25 +0200 Subject: [PATCH 01/11] Use Libtask_jll --- .github/workflows/BuildDylib.yaml | 31 ----------- Project.toml | 6 +- deps/Makefile | 49 ----------------- deps/build.jl | 59 -------------------- deps/build_dylib.sh | 59 -------------------- deps/build_tarballs.jl | 75 ------------------------- deps/task.c | 91 ------------------------------- src/Libtask.jl | 13 +---- 8 files changed, 5 insertions(+), 378 deletions(-) delete mode 100644 .github/workflows/BuildDylib.yaml delete mode 100644 deps/Makefile delete mode 100644 deps/build.jl delete mode 100644 deps/build_dylib.sh delete mode 100644 deps/build_tarballs.jl delete mode 100644 deps/task.c diff --git a/.github/workflows/BuildDylib.yaml b/.github/workflows/BuildDylib.yaml deleted file mode 100644 index 4427a47a..00000000 --- a/.github/workflows/BuildDylib.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build Dylib - -on: - push: - paths: - - 'deps/Makefile' - - 'deps/task.c' - - 'deps/build_tarballs.jl' - - 'deps/build_dylib.sh' - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - julia-version: [1.2] - julia-arch: [x64] - os: [ubuntu-latest] - steps: - - uses: actions/checkout@v1.0.0 - - name: Set up Julia - uses: julia-actions/setup-julia@latest - with: - version: ${{ matrix.julia-version }} - - name: Install dependencies - run: julia -e 'using Pkg; Pkg.add("BinaryProvider"); Pkg.add("BinaryBuilder");' - - name: Build - env: - BINARYBUILDER_DOWNLOADS_CACHE: downloads - BINARYBUILDER_AUTOMATIC_APPLE: true - run: julia deps/build_tarballs.jl diff --git a/Project.toml b/Project.toml index d0044127..ec677761 100644 --- a/Project.toml +++ b/Project.toml @@ -6,13 +6,13 @@ repo = "https://github.com/TuringLang/Libtask.jl.git" version = "0.4.2" [deps] -BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Libtask_jll = "3ae2931a-708c-5973-9c38-ccf7496fb450" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] -BinaryProvider = "0.5" -julia = "1" +Libtask_jll = "0.3.2" +julia = "1.3" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/deps/Makefile b/deps/Makefile deleted file mode 100644 index 2c09436c..00000000 --- a/deps/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -JULIA_HOME ?= $(PWD) -target ?= x86_64-linux-gnu -prefix ?= $(PWD) -JULIA_VERSION ?= v1_1 - -INCLUDES = -I$(JULIA_HOME)/include/julia - -ifneq (,$(findstring mingw32,$(target))) # mingw32 - OUTPUT_DIR = $(prefix)/bin -else - OUTPUT_DIR = $(prefix)/lib -endif - - -ifneq (,$(findstring mingw32,$(target))) # mingw32 - LIBS = -L$(JULIA_HOME)/bin -else - LIBS = -L$(JULIA_HOME)/lib -L$(JULIA_HOME)/lib/julia -endif - - -LDFLAGS = $(LIBS) -ljulia -ifneq (,$(findstring mingw32,$(target))) # mingw32 - LDFLAGS += -lopenlibm -endif - -CFLAGS = -O2 -shared -std=gnu99 -DJULIA_ENABLE_THREADING=1 -fPIC -ifneq (,$(findstring i686,$(target))) # i686 - CFLAGS += -march=pentium4 -endif -ifneq (,$(findstring linux,$(target))) # linux - CFLAGS += -Wl,--export-dynamic -endif -ifneq (,$(findstring mingw32,$(target))) # mingw32 - CFLAGS += -Wl,--export-all-symbols -endif - -ifneq (,$(findstring mingw32,$(target))) # mingw32 - DYEXT = dll -else ifneq (,$(findstring darwin,$(target))) # macOS - DYEXT = dylib -else - DYEXT = so -endif - - -all: - mkdir -p $(OUTPUT_DIR) - $(CC) $(CFLAGS) $(INCLUDES) $(LDFLAGS) task.c -o $(OUTPUT_DIR)/libtask_$(JULIA_VERSION).$(DYEXT) diff --git a/deps/build.jl b/deps/build.jl deleted file mode 100644 index b58575dc..00000000 --- a/deps/build.jl +++ /dev/null @@ -1,59 +0,0 @@ -### -### This file is generated by running -### ` julia generate_buildjl.jl L/Libtask/build_tarballs.jl` -### in the Yggdrasil root directory, with 2 updates: -### 1. add prefix tp products, see https://github.com/JuliaPackaging/Yggdrasil#binaryproviderjl, -### 2. products filter -### - -using BinaryProvider # requires BinaryProvider 0.3.0 or later - -# Parse some basic command-line arguments -const verbose = "--verbose" in ARGS -const prefix = Prefix(get([a for a in ARGS if a != "--verbose"], 1, joinpath(@__DIR__, "usr"))) -products = [ - LibraryProduct(prefix, ["libtask_v1_0"], :libtask_v1_0), - LibraryProduct(prefix, ["libtask_v1_1"], :libtask_v1_1), - LibraryProduct(prefix, ["libtask_v1_2"], :libtask_v1_2), - LibraryProduct(prefix, ["libtask_v1_3"], :libtask_v1_3), -] - -products_tmp = filter(products) do prod - endswith(prod.libnames[1], "$(VERSION.major)_$(VERSION.minor)") -end -length(products_tmp) == 0 && (products_tmp = [products[end]]) -products = products_tmp - -# Download binaries from hosted location -bin_prefix = "https://github.com/JuliaBinaryWrappers/Libtask_jll.jl/releases/download/Libtask-v0.3.2+0" - -# Listing of files generated by BinaryBuilder: -download_info = Dict( - Linux(:aarch64, libc=:glibc) => ("$bin_prefix/Libtask.v0.3.2.aarch64-linux-gnu.tar.gz", "11fb468221a7da7fe8aedfa7701296207362870ced59a1d0b5c9988cd3697830"), - Linux(:i686, libc=:glibc) => ("$bin_prefix/Libtask.v0.3.2.i686-linux-gnu.tar.gz", "911ac6d3f5340ece7bc55f06bed880d4ca44520c5d55b50dcf722853b4072559"), - Windows(:i686) => ("$bin_prefix/Libtask.v0.3.2.i686-w64-mingw32.tar.gz", "3dd17460d7bdeb9914e0a67b6517a998f456a5fc51e07e3d584cd0ad23f5ff22"), - Linux(:powerpc64le, libc=:glibc) => ("$bin_prefix/Libtask.v0.3.2.powerpc64le-linux-gnu.tar.gz", "ab6bdb03193763800c3fabd108d851718b089c498e30950118cb771559c75039"), - MacOS(:x86_64) => ("$bin_prefix/Libtask.v0.3.2.x86_64-apple-darwin14.tar.gz", "9eb6c3492444103492cfcb7c6ced082c558574b5c6e57577e3a48385a14038e6"), - Linux(:x86_64, libc=:glibc) => ("$bin_prefix/Libtask.v0.3.2.x86_64-linux-gnu.tar.gz", "c8f0a224ef342345ea1002a6726d29969b9236f4d3b1427999a56fe2e71b39fe"), - Windows(:x86_64) => ("$bin_prefix/Libtask.v0.3.2.x86_64-w64-mingw32.tar.gz", "fb1b060d94b955107e384c033db20c4ea23ed0265165e47e6ea5e0d68fa56093"), -) - -# Install unsatisfied or updated dependencies: -unsatisfied = any(!satisfied(p; verbose=verbose) for p in products) -dl_info = choose_download(download_info, platform_key_abi()) -if dl_info === nothing && unsatisfied - # If we don't have a compatible .tar.gz to download, complain. - # Alternatively, you could attempt to install from a separate provider, - # build from source or something even more ambitious here. - error("Your platform (\"$(Sys.MACHINE)\", parsed as \"$(triplet(platform_key_abi()))\") is not supported by this package!") -end - -# If we have a download, and we are unsatisfied (or the version we're -# trying to install is not itself installed) then load it up! -if unsatisfied || !isinstalled(dl_info...; prefix=prefix) - # Download and install binaries - install(dl_info...; prefix=prefix, force=true, verbose=verbose) -end - -# Write out a deps.jl file that will contain mappings for our products -write_deps_file(joinpath(@__DIR__, "deps.jl"), products, verbose=verbose) diff --git a/deps/build_dylib.sh b/deps/build_dylib.sh deleted file mode 100644 index 9704469a..00000000 --- a/deps/build_dylib.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# target=x86_64-linux-gnu -cd $WORKSPACE/srcdir - -RELEASES=( - arm-linux-gnueabihf@v1_0@https://julialang-s3.julialang.org/bin/linux/armv7l/1.0/julia-1.0.0-linux-armv7l.tar.gz - arm-linux-gnueabihf@v1_2@https://julialang-s3.julialang.org/bin/linux/armv7l/1.2/julia-1.2.0-linux-armv7l.tar.gz - arm-linux-gnueabihf@v1_3@https://julialang-s3.julialang.org/bin/linux/armv7l/1.3/julia-1.3.0-linux-armv7l.tar.gz - aarch64-linux-gnu@v1_0@https://julialang-s3.julialang.org/bin/linux/aarch64/1.0/julia-1.0.4-linux-aarch64.tar.gz - aarch64-linux-gnu@v1_1@https://julialang-s3.julialang.org/bin/linux/aarch64/1.1/julia-1.1.1-linux-aarch64.tar.gz - aarch64-linux-gnu@v1_2@https://julialang-s3.julialang.org/bin/linux/aarch64/1.2/julia-1.2.0-linux-aarch64.tar.gz - aarch64-linux-gnu@v1_3@https://julialang-s3.julialang.org/bin/linux/aarch64/1.3/julia-1.3.0-linux-aarch64.tar.gz - i686-linux-gnu@v1_0@https://julialang-s3.julialang.org/bin/linux/x86/1.0/julia-1.0.0-linux-i686.tar.gz - i686-linux-gnu@v1_1@https://julialang-s3.julialang.org/bin/linux/x86/1.1/julia-1.1.0-linux-i686.tar.gz - i686-linux-gnu@v1_2@https://julialang-s3.julialang.org/bin/linux/x86/1.2/julia-1.2.0-linux-i686.tar.gz - i686-linux-gnu@v1_3@https://julialang-s3.julialang.org/bin/linux/x86/1.3/julia-1.3.0-linux-i686.tar.gz - x86_64-linux-gnu@v1_0@https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.0-linux-x86_64.tar.gz - x86_64-linux-gnu@v1_1@https://julialang-s3.julialang.org/bin/linux/x64/1.1/julia-1.1.0-linux-x86_64.tar.gz - x86_64-linux-gnu@v1_2@https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2.0-linux-x86_64.tar.gz - x86_64-linux-gnu@v1_3@https://julialang-s3.julialang.org/bin/linux/x64/1.3/julia-1.3.0-linux-x86_64.tar.gz - powerpc64le-linux-gnu@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-linux-ppc64le.tar.gz - powerpc64le-linux-gnu@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-linux-ppc64le.tar.gz - powerpc64le-linux-gnu@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-linux-ppc64le.tar.gz - powerpc64le-linux-gnu@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-linux-ppc64le.tar.gz - x86_64-w64-mingw32@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-win64.tar.gz - x86_64-w64-mingw32@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-win64.tar.gz - x86_64-w64-mingw32@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-win64.tar.gz - x86_64-w64-mingw32@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-win64.tar.gz - i686-w64-mingw32@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-win32.tar.gz - i686-w64-mingw32@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-win32.tar.gz - i686-w64-mingw32@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-win32.tar.gz - i686-w64-mingw32@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-win32.tar.gz - x86_64-apple-darwin14@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-mac64.tar.gz - x86_64-apple-darwin14@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-mac64.tar.gz - x86_64-apple-darwin14@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-mac64.tar.gz - x86_64-apple-darwin14@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-mac64.tar.gz -) - -for RELEASE in ${RELEASES[@]}; do - REL_TARGET=$(echo $RELEASE | cut -d@ -f1) - if [ $target != $REL_TARGET ]; then - continue - fi - rm -f *.tar.gz - rm -fr julia - wget $(echo $RELEASE | cut -d@ -f3) - tar xzvf *tar.gz - rm -f *.tar.gz - mv julia* julia - export JULIA_VERSION=$(echo $RELEASE | cut -d@ -f2) - if [ $target = x86_64-apple-darwin14 ]; then - export JULIA_HOME=$(pwd)/julia/Contents/Resources/julia - else - export JULIA_HOME=$(pwd)/julia - # export JULIA_HOME=/home/kdr2/programs/julia-1.1.0 - fi - make -C Libtask.jl/deps -done diff --git a/deps/build_tarballs.jl b/deps/build_tarballs.jl deleted file mode 100644 index 39d2d3f3..00000000 --- a/deps/build_tarballs.jl +++ /dev/null @@ -1,75 +0,0 @@ -# Note that this script can accept some limited command-line arguments, run -# `julia build_tarballs.jl --help` to see a usage message. -using Pkg -using BinaryBuilder - -name = "LibtaskDylib" -version_str = Pkg.TOML.parsefile(joinpath(@__DIR__, "../Project.toml"))["version"] |> strip |> (x) -> lstrip(x, ['v']) -version = VersionNumber(version_str) -# see https://github.com/JuliaPackaging/BinaryBuilder.jl/issues/336 -ENV["CI_COMMIT_TAG"] = ENV["TRAVIS_TAG"] = "v" * version_str - -event_file = get(ENV, "GITHUB_EVENT_PATH", "") -# run(`cat $event_file`) - -# Collection of sources required to build Libtask -function get_commit_id() - is_pr = get(ENV, "TRAVIS_PULL_REQUEST", "false") - if is_pr != "false" - return get(ENV, "TRAVIS_PULL_REQUEST_SHA", "") - end - - ref = "HEAD" - - gaction = get(ENV, "GITHUB_ACTIONS", "") - if !isempty(gaction) - # .pull_request.head.sha, .release.tag_name, - ref = readlines(`jq --raw-output '.pull_request.head.sha' $event_file`)[1] - if ref == "null" - ref = readlines(`jq --raw-output '.release.tag_name' $event_file`)[1] - end - end - - if ref == "null" - ref = "HEAD" - end - return readlines(`git rev-parse $ref`)[1] -end - -sources = [ - "https://github.com/TuringLang/Libtask.jl.git" => get_commit_id(), -] - - -# Bash recipe for building across all platforms -script = read(joinpath(dirname(@__FILE__), "build_dylib.sh"), String) - -# These are the platforms we will build for by default, unless further -# platforms are passed in on the command line -platforms = [ - Linux(:i686, :glibc), - Linux(:x86_64, :glibc), - Linux(:powerpc64le, libc=:glibc), - # Linux(:armv7l, :glibc, :eabihf), - Linux(:aarch64), - MacOS(:x86_64), - Windows(:i686), - Windows(:x86_64) -] - -# The products that we will ensure are always built -products(prefix) = [ - LibraryProduct(prefix, "libtask_v1_0", :libtask_v1_0) - LibraryProduct(prefix, "libtask_v1_1", :libtask_v1_1) - LibraryProduct(prefix, "libtask_v1_2", :libtask_v1_2) - LibraryProduct(prefix, "libtask_v1_3", :libtask_v1_3) -] - -# Dependencies that must be installed before this package can be built -dependencies = [ - -] - -# Build the tarballs, and possibly a `build.jl` as well. -# build_file = "products/build_$(name).v$(version_str).jl" -build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies) diff --git a/deps/task.c b/deps/task.c deleted file mode 100644 index 7eb16eb9..00000000 --- a/deps/task.c +++ /dev/null @@ -1,91 +0,0 @@ -/* - task.c - task copying (aka continuation) for lightweight processes (symmetric coroutines) - */ - -#include "julia.h" - -// Workaround for JuliaLang/julia#32812 -jl_task_t *vanilla_get_current_task(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - return (jl_task_t*)ptls->current_task; -} - - -jl_task_t *jl_enable_stack_copying(jl_task_t *t) -{ -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1 - if (!t->copy_stack) { - jl_ptls_t ptls = jl_get_ptls_states(); - t->copy_stack = 1; - t->bufsz = 0; - memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx)); - } -#endif - return t; -} - -jl_task_t *jl_clone_task(jl_task_t *t) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - jl_task_t *newt = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); // More efficient - //jl_task_t *newt = (jl_task_t*)jl_new_task(t->start, t->ssize); // Less efficient - memset(newt, 0, sizeof(jl_task_t)); - jl_set_typeof(newt, jl_task_type); - newt->stkbuf = NULL; - newt->gcstack = NULL; - JL_GC_PUSH1(&newt); - - newt->state = t->state; - newt->start = t->start; - newt->tls = jl_nothing; - newt->logstate = ptls->current_task->logstate; - newt->result = jl_nothing; - newt->donenotify = jl_nothing; - newt->exception = jl_nothing; - newt->backtrace = jl_nothing; - newt->eh = t->eh; - newt->gcstack = t->gcstack; - newt->tid = t->tid; // TODO: need testing - newt->started = t->started; // TODO: need testing - - -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1 - newt->copy_stack = t->copy_stack; - memcpy((void*)newt->ctx.uc_mcontext, (void*)t->ctx.uc_mcontext, sizeof(jl_jmp_buf)); -#if JULIA_VERSION_MINOR >= 2 - newt->queue = t->queue; - newt->sticky = t->sticky; -#endif -#else - newt->parent = ptls->current_task; - newt->current_module = t->current_module; - newt->ssize = t->ssize; // size of saved piece - memcpy((void*)newt->ctx, (void*)t->ctx, sizeof(jl_jmp_buf)); -#endif - - if (t->stkbuf) { -#if !(JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1) - newt->ssize = t->ssize; // size of saved piece -#endif - // newt->stkbuf = allocb(t->bufsz); - // newt->bufsz = t->bufsz; - // memcpy(newt->stkbuf, t->stkbuf, t->bufsz); - // workaround, newt and t will get new stkbuf when savestack is called. - t->bufsz = 0; - newt->bufsz = 0; - newt->stkbuf = t->stkbuf; - } else { -#if !(JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1) - newt->ssize = 0; -#endif - newt->bufsz = 0; - newt->stkbuf = NULL; - } - - JL_GC_POP(); - jl_gc_wb_back(newt); - - return newt; -} diff --git a/src/Libtask.jl b/src/Libtask.jl index f0dec691..c21403fa 100644 --- a/src/Libtask.jl +++ b/src/Libtask.jl @@ -1,17 +1,8 @@ module Libtask -export CTask, consume, produce, TArray, get, tzeros, tfill, TRef - -# Try to load the binary dependency -const depsjl_path = joinpath(@__DIR__, "..", "deps", "deps.jl") -if !isfile(depsjl_path) - error("Libtask is not properly installed. Please run `import Pkg; Pkg.build(\"Libtask\")`") -end -include(depsjl_path) +using Libtask_jll -function __init__() - check_deps() -end +export CTask, consume, produce, TArray, get, tzeros, tfill, TRef @static if VERSION < v"1.0.9999" # (-, v1.1) const libtask = libtask_v1_0 From 3bff974acc3e4c6a51f6156b36a00d56344f5844 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 17:43:47 +0200 Subject: [PATCH 02/11] Update README --- README.md | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 47ea7ffe..731d7a5d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Libtask -[![Libtask Testing](https://github.com/TuringLang/Libtask.jl/workflows/Libtask%20Testing/badge.svg)](https://github.com/TuringLang/Libtask.jl/actions) -[![Dylib Build](https://github.com/TuringLang/Libtask.jl/workflows/Build%20Dylib/badge.svg)](https://github.com/TuringLang/Libtask.jl/actions) +[![Libtask Testing](https://github.com/TuringLang/Libtask.jl/workflows/Libtask%20Testing/badge.svg)](https://github.com/TuringLang/Libtask.jl/actions?branch=master) C shim for [task copying](https://github.com/JuliaLang/julia/issues/4085) in Turing @@ -90,28 +89,6 @@ a = copy(ctask) Note: The [Turing](https://github.com/TuringLang/Turing.jl) probabilistic programming language uses this task copying feature in an efficient implementation of the [particle filtering](https://en.wikipedia.org/wiki/Particle_filter) sampling algorithm for arbitary order [Markov processes](https://en.wikipedia.org/wiki/Markov_model#Hidden_Markov_model). -Disclaimer: This feature is still experimental and should only be used with caution. Some discussions on its potential caveats can be found [here](https://github.com/JuliaLang/julia/pull/15078). - -## For Developer - -### Release a new version - - If you have updates of the binary dylib (files under the `deps` - directory), you should: - - Update the version/commit information in [this - file](https://github.com/JuliaPackaging/Yggdrasil/blob/master/L/Libtask/build_tarballs.jl) - by a pull request - - Wait for that PR being merged, then you will find a new - release of the dylib - [here](https://github.com/JuliaBinaryWrappers/Libtask_jll.jl/releases) - - Then, in the root directory of the Yggdrasil project, run - `https://github.com/JuliaBinaryWrappers/Libtask_jll.jl/releases`, - you will get a build file like - `build/build_Libtask.v0.3.1.jl`. Copy the binary download - information (the variables `bin_prefix` and `download_info`) - from that generated file to `deps/build.jl` in our repo and - commit the changes. - - If you don't make any changes about the dylib, just ignore the - first step and go to next one. - - Update the new version number in `Project.toml`, ping - @JuliaRegistrator with `@JuliaRegistrator register` in a - comment. +## Disclaimer + +This feature is still experimental and should only be used with caution. Some discussions on its potential caveats can be found [here](https://github.com/JuliaLang/julia/pull/15078). From ced6702a16ac600a63d34f424c1d91b12123f41a Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 17:44:00 +0200 Subject: [PATCH 03/11] Add CompatHelper --- .github/workflows/CompatHelper.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/CompatHelper.yml diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 00000000..952dc45d --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: '00 00 * * *' + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }} + run: julia -e 'using CompatHelper; CompatHelper.main()' \ No newline at end of file From 8cf3c936704d94645d8433a2f3579122eac5bd5a Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 17:44:10 +0200 Subject: [PATCH 04/11] Bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index ec677761..113a0d3c 100644 --- a/Project.toml +++ b/Project.toml @@ -3,7 +3,7 @@ uuid = "6f1fad26-d15e-5dc8-ae53-837a1d7b8c9f" license = "MIT" desc = "C shim for task copying in Turing" repo = "https://github.com/TuringLang/Libtask.jl.git" -version = "0.4.2" +version = "0.5.0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From b99ef70b3361f68d602d99d419628b463a9778de Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 17:52:03 +0200 Subject: [PATCH 05/11] Do not test old Julia versions --- .github/workflows/Testing.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/Testing.yaml b/.github/workflows/Testing.yaml index 4aaaf7c8..cfac97bb 100644 --- a/.github/workflows/Testing.yaml +++ b/.github/workflows/Testing.yaml @@ -15,11 +15,9 @@ jobs: strategy: matrix: version: - - '1.0' - - '1.1' - - '1.2' - '1.3' - '1.4' + - '1' - 'nightly' os: - ubuntu-latest From 5d86aaa4e8f5b83cc4187979b4c6e80cbb23674a Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 17:58:25 +0200 Subject: [PATCH 06/11] Do not export `get` --- src/Libtask.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libtask.jl b/src/Libtask.jl index c21403fa..bd270cbb 100644 --- a/src/Libtask.jl +++ b/src/Libtask.jl @@ -2,7 +2,7 @@ module Libtask using Libtask_jll -export CTask, consume, produce, TArray, get, tzeros, tfill, TRef +export CTask, consume, produce, TArray, tzeros, tfill, TRef @static if VERSION < v"1.0.9999" # (-, v1.1) const libtask = libtask_v1_0 From 25cbfc0afb8fdfca985c1034aea0e2ae4d76251c Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 9 Oct 2020 18:17:46 +0200 Subject: [PATCH 07/11] Clean list of dependencies --- Project.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Project.toml b/Project.toml index 113a0d3c..2ea02a34 100644 --- a/Project.toml +++ b/Project.toml @@ -6,9 +6,7 @@ repo = "https://github.com/TuringLang/Libtask.jl.git" version = "0.5.0" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Libtask_jll = "3ae2931a-708c-5973-9c38-ccf7496fb450" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] Libtask_jll = "0.3.2" From 62ebc46fd00ecd3039cb231cc882fb6b5296d376 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 20 Nov 2020 17:20:11 +0100 Subject: [PATCH 08/11] Update to Libtask_jll 0.4 --- Project.toml | 2 +- src/Libtask.jl | 10 -------- src/ctask.jl | 69 +++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index 2ea02a34..ad8fe70e 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,7 @@ version = "0.5.0" Libtask_jll = "3ae2931a-708c-5973-9c38-ccf7496fb450" [compat] -Libtask_jll = "0.3.2" +Libtask_jll = "0.4" julia = "1.3" [extras] diff --git a/src/Libtask.jl b/src/Libtask.jl index bd270cbb..e647e65d 100644 --- a/src/Libtask.jl +++ b/src/Libtask.jl @@ -4,16 +4,6 @@ using Libtask_jll export CTask, consume, produce, TArray, tzeros, tfill, TRef -@static if VERSION < v"1.0.9999" # (-, v1.1) - const libtask = libtask_v1_0 -elseif VERSION < v"1.1.9999" # [v1.1, v1.2) - const libtask = libtask_v1_1 -elseif VERSION < v"1.2.9999" # [v1.2, v1.3) - const libtask = libtask_v1_2 -else # [v1.3, +) - const libtask = libtask_v1_3 -end - include("ctask.jl") include("tarray.jl") include("tref.jl") diff --git a/src/ctask.jl b/src/ctask.jl index d5cb8d1b..0948ee2f 100644 --- a/src/ctask.jl +++ b/src/ctask.jl @@ -24,11 +24,11 @@ end function Base.showerror(io::IO, ex::CTaskException) println(io, "CTaskException:") - showerror(io, ex.task.exception, ex.task.backtrace) + showerror(io, getproperty(ex.task, :exception), getproperty(ex.task, :backtrace)) end # Utility function for self-copying mechanism -_current_task() = ccall((:vanilla_get_current_task, libtask), Ref{Task}, ()) +_current_task() = ccall((:vanilla_get_current_task, libtask_julia), Ref{Task}, ()) n_copies() = n_copies(_current_task()) function n_copies(t::Task) @@ -37,10 +37,11 @@ function n_copies(t::Task) end function enable_stack_copying(t::Task) - if t.state !== :runnable && t.state !== :done + state = getproperty(t, :state) + if state !== :runnable && state !== :done error("only runnable or finished tasks' stack can be copied.") end - return ccall((:jl_enable_stack_copying, libtask), Any, (Any,), t)::Task + return ccall((:jl_enable_stack_copying, libtask_julia), Any, (Any,), t)::Task end """ @@ -69,7 +70,11 @@ function task_wrapper(func) wait() catch ex ct = _current_task() - ct.exception = ex + @static if VERSION < v"1.6.0-DEV.1145" + ct.exception = ex + else + ct._isexception = true + end ct.result = ex ct.backtrace = catch_backtrace() ct.storage === nothing && (ct.storage = IdDict()) @@ -83,18 +88,19 @@ end function Base.copy(ctask::CTask) task = ctask.task - if task.state !== :runnable && task.state !== :done + state = getproperty(task, :state) + if state !== :runnable && state !== :done error("only runnable or finished tasks can be copied.") end - newtask = ccall((:jl_clone_task, libtask), Any, (Any,), task)::Task + newtask = ccall((:jl_clone_task, libtask_julia), Any, (Any,), task)::Task task.storage[:n_copies] = 1 + n_copies(task) newtask.storage = copy(task.storage) # copy fields not accessible in task.c newtask.code = task.code - newtask.state = task.state + setstate!(newtask, getstate(task)) newtask.result = task.result @static if VERSION < v"1.1" newtask.parent = task.parent @@ -133,7 +139,7 @@ function produce(v) end # Internal check to make sure that it is possible to switch to the consumer. - @assert task.state in (:runnable, :queued) + @assert getproperty(task, :state) in (:runnable, :queued) @static if VERSION < v"1.1.9999" task.state === :queued && yield() @@ -197,22 +203,55 @@ function consume(ctask::CTask, values...) push!(producer.storage[:consumers].waitq, ct) end - if producer.state === :runnable + if getproperty(producer, :state) === :runnable # Switch to the producer. schedule(producer) yield() # Update the state if possible. if producer.storage isa IdDict && haskey(producer.storage, :_libtask_state) - producer.state = producer.storage[:_libtask_state] + setstate!(producer, producer.storage[:_libtask_state]) end - # If the task is done return the result. - producer.state === :done && return producer.result - # If the task failed, throw an exception. - producer.state === :failed && throw(CTaskException(producer)) + _istaskfailed(producer) && throw(CTaskException(producer)) + + # If the task is done return the result. + istaskdone(producer) && return producer.result end wait() end + +function _istaskfailed(task::Task) + @static if VERSION < v"1.3" + return task.state === :failed + else + return Base.istaskfailed(task) + end +end + +function getstate(task::Task) + @static if VERSION < v"1.6.0-DEV.618" + return task.state + else + return task._state + end +end + +function setstate!(task::Task, state) + @static if VERSION < v"1.6.0-DEV.618" + task.state = state + else + if state === :runnable + task._state = Base.task_state_runnable + elseif state === :done + task._state = Base.task_state_done + elseif state === :failed + task._state = Base.task_state_failed + else + task._state = state + end + end +end + From df9a85955ca2ea448f45416f64b62d013a859114 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Mon, 23 Nov 2020 17:37:40 +0100 Subject: [PATCH 09/11] Add Libtask_jll manually on Julia nightly --- .github/workflows/TagBot.yml | 9 ++++++--- .github/workflows/Testing.yaml | 7 ++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index ac8d9bb4..778c06fe 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -1,11 +1,14 @@ name: TagBot on: - schedule: - - cron: 0 * * * * + issue_comment: + types: + - created + workflow_dispatch: jobs: TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' runs-on: ubuntu-latest steps: - uses: JuliaRegistries/TagBot@v1 with: - token: ${{ secrets.DEPLOY_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/Testing.yaml b/.github/workflows/Testing.yaml index cfac97bb..89386607 100644 --- a/.github/workflows/Testing.yaml +++ b/.github/workflows/Testing.yaml @@ -1,13 +1,9 @@ name: Libtask Testing - on: push: branches: - master - # tags: '*' - release: pull_request: - jobs: test: runs-on: ${{ matrix.os }} @@ -29,12 +25,13 @@ jobs: exclude: - os: macOS-latest arch: x86 - steps: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} + - if: ${{ matrix.version == 'nightly' }} + run: julia -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/JuliaBinaryWrappers/Libtask_jll.jl.git"))' - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-runtest@latest From 4bfa5b5c2bf34288d35977ddc971f33521272311 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Mon, 23 Nov 2020 17:46:39 +0100 Subject: [PATCH 10/11] Add cache --- .github/workflows/Testing.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Testing.yaml b/.github/workflows/Testing.yaml index 89386607..8d6d66e7 100644 --- a/.github/workflows/Testing.yaml +++ b/.github/workflows/Testing.yaml @@ -31,7 +31,17 @@ jobs: with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- - if: ${{ matrix.version == 'nightly' }} - run: julia -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/JuliaBinaryWrappers/Libtask_jll.jl.git"))' + run: julia --color=yes --project -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/JuliaBinaryWrappers/Libtask_jll.jl.git"))' - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-runtest@latest From bdbf6b1162c9df30f8c898a67d3f54faed41b3cb Mon Sep 17 00:00:00 2001 From: David Widmann Date: Mon, 23 Nov 2020 17:58:01 +0100 Subject: [PATCH 11/11] Add section to README --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 731d7a5d..36ae19b9 100644 --- a/README.md +++ b/README.md @@ -92,3 +92,15 @@ Note: The [Turing](https://github.com/TuringLang/Turing.jl) probabilistic progra ## Disclaimer This feature is still experimental and should only be used with caution. Some discussions on its potential caveats can be found [here](https://github.com/JuliaLang/julia/pull/15078). + +## Julia nightly + +Libtask uses the `libtask_julia` library which is pre-built for Julia versions 1.3, 1.4, and 1.5 and +distributed via the [Libtask_jll](https://github.com/JuliaBinaryWrappers/Libtask_jll.jl/) package. + +Julia nightly might not be compatible with the latest version of the `libtask_julia` library and is +not officially supported. If you want to use Julia nightly, you have to add the Libtask_jll package +manually: +```julia +julia> ] add https://github.com/JuliaBinaryWrappers/Libtask_jll.jl.git +```