diff --git a/.travis.yml b/.travis.yml index b3e53eea..99cd101b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: julia +dist: trusty # update "python3.4-venv" below when updating os: - linux julia: @@ -10,6 +11,7 @@ addons: packages: - python-numpy - python3-numpy + - python3.4-venv # Ubuntu Trusty 14.04 does not have python3-venv env: global: - PYCALL_DEBUG_BUILD="yes" diff --git a/README.md b/README.md index 444de9c1..0092426a 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,12 @@ unless you set the `PYTHON` environment variable or delete the file **Note:** If you use Python [virtualenvs](http://docs.python-guide.org/en/latest/dev/virtualenvs/), -then be aware that PyCall *uses the virtualenv it was built with*, even -if you switch virtualenvs. If you want to switch PyCall to use a -different virtualenv, then you should switch virtualenvs and run -`rm(Pkg.dir("PyCall","deps","PYTHON")); Pkg.build("PyCall")`. +then be aware that PyCall *uses the virtualenv it was built with* by +default, even if you switch virtualenvs. If you want to switch PyCall +to use a different virtualenv, then you should switch virtualenvs and +run `rm(Pkg.dir("PyCall","deps","PYTHON")); Pkg.build("PyCall")`. +Alternatively, see [Python virtual environments](#python-virtual-environments) +section below for switching virtual environment at run-time. **Note:** Usually, the necessary libraries are installed along with Python, but [pyenv on MacOS](https://github.com/JuliaPy/PyCall.jl/issues/122) @@ -543,6 +545,35 @@ and so on. Here, instead of `pyimport`, we have used the function `pyimport_conda`. The second argument is the name of the [Anaconda package](https://docs.continuum.io/anaconda/pkg-docs) that provides this module. This way, if importing `scipy.optimize` fails because the user hasn't installed `scipy`, it will either (a) automatically install `scipy` and retry the `pyimport` if PyCall is configured to use the [Conda](https://github.com/Luthaf/Conda.jl) Python install (or any other Anaconda-based Python distro for which the user has installation privileges), or (b) throw an error explaining that `scipy` needs to be installed, and explain how to configure PyCall to use Conda so that it can be installed automatically. More generally, you can call `pyimport(module, package, channel)` to specify an optional Anaconda "channel" for installing non-standard Anaconda packages. +## Python virtual environments + +Python virtual environments created by [`venv`](https://docs.python.org/3/library/venv.html) and [`virtualenv`](https://virtualenv.pypa.io/en/latest/) +can be used from `PyCall`, *provided that the Python executable used +in the virtual environment is linked against the same `libpython` used +by `PyCall`*. Note that virtual environments created by `conda` are not +supported. + +To use `PyCall` with a certain virtual environment, set the environment +variable `PYCALL_JL_RUNTIME_PYTHON` *before* importing `PyCall` to +path to the Python executable. For example: + +```julia +$ source PATH/TO/bin/activate # activate virtual environment in system shell + +$ julia # start Julia +... + +julia> ENV["PYCALL_JL_RUNTIME_PYTHON"] = Sys.which("python") +"PATH/TO/bin/python3" + +julia> using PyCall + +julia> pyimport("sys").executable +"PATH/TO/bin/python3" +``` +Similarly, the `PYTHONHOME` path can be changed by the environment variable +`PYCALL_JL_RUNTIME_PYTHONHOME`. + ## Author This package was written by [Steven G. Johnson](http://math.mit.edu/~stevenj/). diff --git a/deps/build.jl b/deps/build.jl index 0ae93206..5c526781 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -12,28 +12,6 @@ struct UseCondaPython <: Exception end ######################################################################### -# Fix the environment for running `python`, and setts IO encoding to UTF-8. -# If cmd is the Conda python, then additionally removes all PYTHON* and -# CONDA* environment variables. -function pythonenv(cmd::Cmd) - env = copy(ENV) - if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR) - pythonvars = String[] - for var in keys(env) - if startswith(var, "CONDA") || startswith(var, "PYTHON") - push!(pythonvars, var) - end - end - for var in pythonvars - pop!(env, var) - end - end - # set PYTHONIOENCODING when running python executable, so that - # we get UTF-8 encoded text as output (this is not the default on Windows). - env["PYTHONIOENCODING"] = "UTF-8" - setenv(cmd, env) -end - pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = chomp(read(pythonenv(`$python -c "import $mod; print($mod.$var)"`), String)) pyconfigvar(python::AbstractString, var::AbstractString) = pyvar(python, "distutils.sysconfig", "get_config_var('$var')") @@ -117,11 +95,6 @@ include("depsutils.jl") ######################################################################### -# A couple of key strings need to be stored as constants so that -# they persist throughout the life of the program. In Python 3, -# they need to be wchar_t* data. -wstringconst(s) = string("Base.cconvert(Cwstring, \"", escape_string(s), "\")") - # we write configuration files only if they change, both # to prevent unnecessary recompilation and to minimize # problems in the unlikely event of read-only directories. @@ -197,15 +170,7 @@ try # make sure deps.jl file is removed on error # Get PYTHONHOME, either from the environment or from Python # itself (if it is not in the environment or if we are using Conda) PYTHONHOME = if !haskey(ENV, "PYTHONHOME") || use_conda - # PYTHONHOME tells python where to look for both pure python - # and binary modules. When it is set, it replaces both - # `prefix` and `exec_prefix` and we thus need to set it to - # both in case they differ. This is also what the - # documentation recommends. However, they are documented - # to always be the same on Windows, where it causes - # problems if we try to include both. - exec_prefix = pysys(python, "exec_prefix") - Sys.iswindows() ? exec_prefix : pysys(python, "prefix") * ":" * exec_prefix + pythonhome_of(python) else ENV["PYTHONHOME"] end @@ -223,10 +188,8 @@ try # make sure deps.jl file is removed on error const python = "$(escape_string(python))" const libpython = "$(escape_string(libpy_name))" const pyprogramname = "$(escape_string(programname))" - const wpyprogramname = $(wstringconst(programname)) const pyversion_build = $(repr(pyversion)) const PYTHONHOME = "$(escape_string(PYTHONHOME))" - const wPYTHONHOME = $(wstringconst(PYTHONHOME)) "True if we are using the Python distribution in the Conda package." const conda = $use_conda diff --git a/deps/depsutils.jl b/deps/depsutils.jl index ba03ec0c..6bb11df5 100644 --- a/deps/depsutils.jl +++ b/deps/depsutils.jl @@ -13,21 +13,143 @@ function findsym(lib, syms...) error("no symbol found from: ", syms) end +# Static buffer to make sure the string passed to libpython persists +# for the lifetime of the program, as CPython API requires: +const __buf_programname = Vector{UInt8}(undef, 1024) +const __buf_pythonhome = Vector{UInt8}(undef, 1024) + # Need to set PythonHome before calling GetVersion to avoid warning (#299). # Unfortunately, this poses something of a chicken-and-egg problem because # we need to know the Python version to set PythonHome via the API. Note # that the string (or array) passed to Py_SetPythonHome needs to be a # constant that lasts for the lifetime of the program, which is why we -# can't use Cwstring here (since that creates a temporary copy). -function Py_SetPythonHome(libpy, PYTHONHOME, wPYTHONHOME, pyversion) - if !isempty(PYTHONHOME) - if pyversion.major < 3 - ccall(Libdl.dlsym(libpy, :Py_SetPythonHome), Cvoid, (Cstring,), PYTHONHOME) - else - ccall(Libdl.dlsym(libpy, :Py_SetPythonHome), Cvoid, (Ptr{Cwchar_t},), wPYTHONHOME) - end +# prepare static buffer __buf_pythonhome, copy the string to it, and then +# pass the pointer to the buffer to the CPython API. +function Py_SetPythonHome(libpy, pyversion, PYTHONHOME::AbstractString) + isempty(PYTHONHOME) && return + if pyversion.major < 3 + ccall(Libdl.dlsym(libpy, :Py_SetPythonHome), Cvoid, (Cstring,), + _preserveas!(__buf_pythonhome, Cstring, PYTHONHOME)) + else + ccall(Libdl.dlsym(libpy, :Py_SetPythonHome), Cvoid, (Ptr{Cwchar_t},), + _preserveas!(__buf_pythonhome, Cwstring, PYTHONHOME)) + end +end + +function Py_SetProgramName(libpy, pyversion, programname::AbstractString) + isempty(programname) && return + if pyversion.major < 3 + ccall(Libdl.dlsym(libpy, :Py_SetProgramName), Cvoid, (Cstring,), + _preserveas!(__buf_programname, Cstring, programname)) + else + ccall(Libdl.dlsym(libpy, :Py_SetProgramName), Cvoid, (Ptr{Cwchar_t},), + _preserveas!(__buf_programname, Cwstring, programname)) end end +""" + _preserveas!(dest::Vector{UInt8}, (Cstring|Cwstring), x::String) :: Ptr + +Copy `x` as `Cstring` or `Cwstring` to `dest` and return a pointer to +`dest`. Thus, this pointer is safe to use as long as `dest` is +protected from GC. +""" +function _preserveas!(dest::Vector{UInt8}, ::Type{Cstring}, x::AbstractString) + s = transcode(UInt8, String(x)) + copyto!(dest, s) + dest[length(s) + 1] = 0 + return pointer(dest) +end + +function _preserveas!(dest::Vector{UInt8}, ::Type{Cwstring}, x::AbstractString) + s = Base.cconvert(Cwstring, x) + copyto!(dest, reinterpret(UInt8, s)) + return pointer(dest) +end + + # need to be able to get the version before Python is initialized Py_GetVersion(libpy) = unsafe_string(ccall(Libdl.dlsym(libpy, :Py_GetVersion), Ptr{UInt8}, ())) + +# Fix the environment for running `python`, and setts IO encoding to UTF-8. +# If cmd is the Conda python, then additionally removes all PYTHON* and +# CONDA* environment variables. +function pythonenv(cmd::Cmd) + env = copy(ENV) + if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR) + pythonvars = String[] + for var in keys(env) + if startswith(var, "CONDA") || startswith(var, "PYTHON") + push!(pythonvars, var) + end + end + for var in pythonvars + pop!(env, var) + end + end + # set PYTHONIOENCODING when running python executable, so that + # we get UTF-8 encoded text as output (this is not the default on Windows). + env["PYTHONIOENCODING"] = "UTF-8" + setenv(cmd, env) +end + + +function pythonhome_of(pyprogramname::AbstractString) + if Sys.iswindows() + # PYTHONHOME tells python where to look for both pure python + # and binary modules. When it is set, it replaces both + # `prefix` and `exec_prefix` and we thus need to set it to + # both in case they differ. This is also what the + # documentation recommends. However, they are documented + # to always be the same on Windows, where it causes + # problems if we try to include both. + script = """ + import sys + if hasattr(sys, "base_exec_prefix"): + sys.stdout.write(sys.base_exec_prefix) + else: + sys.stdout.write(sys.exec_prefix) + """ + else + script = """ + import sys + if hasattr(sys, "base_exec_prefix"): + sys.stdout.write(sys.base_prefix) + sys.stdout.write(":") + sys.stdout.write(sys.base_exec_prefix) + else: + sys.stdout.write(sys.prefix) + sys.stdout.write(":") + sys.stdout.write(sys.exec_prefix) + """ + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME + end + return read(pythonenv(`$pyprogramname -c $script`), String) +end +# To support `venv` standard library (as well as `virtualenv`), we +# need to use `sys.base_prefix` and `sys.base_exec_prefix` here. +# Otherwise, initializing Python in `__init__` below fails with +# unrecoverable error: +# +# Fatal Python error: initfsencoding: unable to load the file system codec +# ModuleNotFoundError: No module named 'encodings' +# +# This is because `venv` does not symlink standard libraries like +# `virtualenv`. For example, `lib/python3.X/encodings` does not +# exist. Rather, `venv` relies on the behavior of Python runtime: +# +# If a file named "pyvenv.cfg" exists one directory above +# sys.executable, sys.prefix and sys.exec_prefix are set to that +# directory and it is also checked for site-packages +# --- https://docs.python.org/3/library/venv.html +# +# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and +# `sys.base_exec_prefix`. If the virtual environment is created by +# `virtualenv`, those `sys.base_*` paths point to the virtual +# environment. Thus, above code supports both use cases. +# +# See also: +# * https://docs.python.org/3/library/venv.html +# * https://docs.python.org/3/library/site.html +# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix +# * https://github.com/JuliaPy/PyCall.jl/issues/410 diff --git a/src/PyCall.jl b/src/PyCall.jl index 30e0d0db..be93a3d5 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -36,6 +36,12 @@ import Base.Iterators: filter include(joinpath(dirname(@__FILE__), "..", "deps","depsutils.jl")) include("startup.jl") +""" +Python executable used by PyCall in the current process. +""" +current_python() = _current_python[] +const _current_python = Ref(pyprogramname) + ######################################################################### # Mirror of C PyObject struct (for non-debugging Python builds). @@ -506,7 +512,7 @@ you want to use, run Pkg.build("PyCall"), and re-launch Julia. msg = msg * """ PyCall is currently configured to use the Python version at: -$python +$(current_python()) and you should use whatever mechanism you usually use (apt-get, pip, conda, etcetera) to install the Python package containing the $name module. diff --git a/src/pyinit.jl b/src/pyinit.jl index 80a4b247..a24995ad 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -61,6 +61,51 @@ function pyjlwrap_init() end end +######################################################################### +# Virtual environment support + +venv_python(::Nothing) = pyprogramname + +function venv_python(venv::AbstractString, suffix::AbstractString = "") + # `suffix` is used to insert version number (e.g., "3.7") in tests + # (see ../test/test_venv.jl) + if Sys.iswindows() + return joinpath(venv, "Scripts", "python$suffix.exe") + else + return joinpath(venv, "bin", "python$suffix") + end + # "Scripts" is used only in Windows and "bin" elsewhere: + # https://github.com/python/cpython/blob/3.7/Lib/venv/__init__.py#L116 +end + +""" + python_cmd(args::Cmd = ``; venv, python) :: Cmd + +Create an appropriate `Cmd` for running Python program with command +line arguments `args`. + +# Keyword Arguments +- `venv::String`: The path of a virtualenv to be used instead of the + default environment with which PyCall isconfigured. +- `python::String`: The path to the Python executable. `venv` is ignored + when this argument is specified. +""" +function python_cmd(args::Cmd = ``; + venv::Union{Nothing, AbstractString} = nothing, + python::AbstractString = venv_python(venv)) + return pythonenv(`$python $args`) +end + +function find_libpython(python::AbstractString) + script = joinpath(@__DIR__, "..", "deps", "find_libpython.py") + cmd = python_cmd(`$script`; python = python) + try + return read(cmd, String) + catch + return nothing + end +end + ######################################################################### const _finalized = Ref(false) @@ -93,14 +138,39 @@ function __init__() already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ()) if !already_inited - Py_SetPythonHome(libpy_handle, PYTHONHOME, wPYTHONHOME, pyversion) - if !isempty(pyprogramname) - if pyversion.major < 3 - ccall((@pysym :Py_SetProgramName), Cvoid, (Cstring,), pyprogramname) + pyhome = PYTHONHOME + + if isfile(get(ENV, "PYCALL_JL_RUNTIME_PYTHON", "")) + _current_python[] = ENV["PYCALL_JL_RUNTIME_PYTHON"] + + # Check libpython compatibility. + venv_libpython = find_libpython(current_python()) + if venv_libpython === nothing + error(""" + `libpython` for $(current_python()) cannot be found. + PyCall.jl cannot initialize Python safely. + """) + elseif venv_libpython != libpython + error(""" + Incompatible `libpython` detected. + `libpython` for $(current_python()) is: + $venv_libpython + `libpython` for $pyprogramname is: + $libpython + PyCall.jl only supports loading Python environment using + the same `libpython`. + """) + end + + if haskey(ENV, "PYCALL_JL_RUNTIME_PYTHONHOME") + pyhome = ENV["PYCALL_JL_RUNTIME_PYTHONHOME"] else - ccall((@pysym :Py_SetProgramName), Cvoid, (Ptr{Cwchar_t},), wpyprogramname) + pyhome = pythonhome_of(current_python()) end end + + Py_SetPythonHome(libpy_handle, pyversion, pyhome) + Py_SetProgramName(libpy_handle, pyversion, current_python()) ccall((@pysym :Py_InitializeEx), Cvoid, (Cint,), 0) end diff --git a/src/startup.jl b/src/startup.jl index 7a8544b2..9fa98e6d 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -46,7 +46,7 @@ if !symbols_present # Only to be used at top-level - pointer will be invalid after reload libpy_handle = Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) # need SetPythonHome to avoid warning, #299 - Py_SetPythonHome(libpy_handle, PYTHONHOME, wPYTHONHOME, pyversion_build) + Py_SetPythonHome(libpy_handle, pyversion_build, PYTHONHOME) else @static if Sys.iswindows() pathbuf = Vector{UInt16}(undef, 1024) diff --git a/test/runtests.jl b/test/runtests.jl index 2628820a..25547852 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -749,3 +749,4 @@ end include("test_pyfncall.jl") include("testpybuffer.jl") +include("test_venv.jl") diff --git a/test/test_venv.jl b/test/test_venv.jl new file mode 100644 index 00000000..01255462 --- /dev/null +++ b/test/test_venv.jl @@ -0,0 +1,123 @@ +using PyCall, Test + + +function test_venv_has_python(path) + newpython = PyCall.python_cmd(venv=path).exec[1] + if !isfile(newpython) + @info """ + Python executable $newpython does not exists. + This directory contains only the following files: + $(join(readdir(dirname(newpython)), '\n')) + """ + end + @test isfile(newpython) +end + + +function test_venv_activation(path) + newpython = PyCall.python_cmd(venv=path).exec[1] + + # Run a fresh Julia process with new Python environment + code = """ + $(Base.load_path_setup_code()) + using PyCall + println(PyCall.pyimport("sys").executable) + println(PyCall.pyimport("sys").exec_prefix) + println(PyCall.pyimport("pip").__file__) + """ + # Note that `pip` is just some arbitrary non-standard + # library. Using standard library like `os` does not work + # because those files are not created. + env = copy(ENV) + env["PYCALL_JL_RUNTIME_PYTHON"] = newpython + jlcmd = setenv(`$(Base.julia_cmd()) --startup-file=no -e $code`, env) + if Sys.iswindows() + # Marking the test broken in Windows. It seems that + # venv copies .dll on Windows and libpython check in + # PyCall.__init__ detects that. + @test_broken begin + output = read(jlcmd, String) + sys_executable, exec_prefix, mod_file = split(output, "\n") + newpython == sys_executable + end + else + output = read(jlcmd, String) + sys_executable, exec_prefix, mod_file = split(output, "\n") + @test newpython == sys_executable + @test startswith(exec_prefix, path) + @test startswith(mod_file, path) + end +end + + +@testset "virtualenv activation" begin + pyname = "python$(pyversion.major).$(pyversion.minor)" + if Sys.which("virtualenv") === nothing + @info "No virtualenv command. Skipping the test..." + elseif Sys.which(pyname) === nothing + @info "No $pyname command. Skipping the test..." + else + mktempdir() do tmppath + if PyCall.pyversion.major == 2 + path = joinpath(tmppath, "kind") + else + path = joinpath(tmppath, "ϵνιℓ") + end + run(`virtualenv --python=$pyname $path`) + test_venv_has_python(path) + + newpython = PyCall.python_cmd(venv=path).exec[1] + venv_libpython = PyCall.find_libpython(newpython) + if venv_libpython != PyCall.libpython + @info """ + virtualenv created an environment with incompatible libpython: + $venv_libpython + """ + return + end + + test_venv_activation(path) + end + end +end + + +@testset "venv activation" begin + # In case PyCall is built with a Python executable created by + # `virtualenv`, let's try to find the original Python executable. + # Otherwise, `venv` does not work with this Python executable: + # https://bugs.python.org/issue30811 + sys = PyCall.pyimport("sys") + if hasproperty(sys, :real_prefix) + # sys.real_prefix is set by virtualenv and does not exist in + # standard Python: + # https://github.com/pypa/virtualenv/blob/16.0.0/virtualenv_embedded/site.py#L554 + candidates = [ + PyCall.venv_python(sys.real_prefix, "$(pyversion.major).$(pyversion.minor)"), + PyCall.venv_python(sys.real_prefix, "$(pyversion.major)"), + PyCall.venv_python(sys.real_prefix), + PyCall.pyprogramname, # must exists + ] + python = candidates[findfirst(isfile, candidates)] + else + python = PyCall.pyprogramname + end + + if PyCall.conda + @info "Skip venv test with conda." + elseif !success(PyCall.python_cmd(`-c "import venv"`, python=python)) + @info "Skip venv test since venv package is missing." + else + mktempdir() do tmppath + if PyCall.pyversion.major == 2 + path = joinpath(tmppath, "kind") + else + path = joinpath(tmppath, "ϵνιℓ") + end + # Create a new virtual environment + run(PyCall.python_cmd(`-m venv $path`, python=python)) + test_venv_has_python(path) + test_venv_activation(path) + end + end +end