diff --git a/NEWS.md b/NEWS.md index b692db7caa6d7b..e0885b713f7aed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -101,6 +101,8 @@ Standard library changes #### Mmap +* `mmap` is now exported ([#39816]). + Deprecated or removed --------------------- diff --git a/stdlib/DelimitedFiles/src/DelimitedFiles.jl b/stdlib/DelimitedFiles/src/DelimitedFiles.jl index b13eae35052135..bf88c12ea9cf2c 100644 --- a/stdlib/DelimitedFiles/src/DelimitedFiles.jl +++ b/stdlib/DelimitedFiles/src/DelimitedFiles.jl @@ -236,7 +236,7 @@ function readdlm_auto(input::AbstractString, dlm::AbstractChar, T::Type, eol::Ab fsz = filesize(input) if use_mmap && fsz > 0 && fsz < typemax(Int) a = open(input, "r") do f - Mmap.mmap(f, Vector{UInt8}, (Int(fsz),)) + mmap(f, Vector{UInt8}, (Int(fsz),)) end # TODO: It would be nicer to use String(a) without making a copy, # but because the mmap'ed array is not NUL-terminated this causes diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 91db2d43701d5b..9a9d795a5aa17a 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -7,6 +7,8 @@ module Mmap import Base: OS_HANDLE, INVALID_OS_HANDLE +export mmap + const PAGESIZE = Int(Sys.isunix() ? ccall(:jl_getpagesize, Clong, ()) : ccall(:jl_getallocationgranularity, Clong, ())) # for mmaps not backed by files @@ -20,7 +22,7 @@ end Mmap.Anonymous(name::AbstractString="", readonly::Bool=false, create::Bool=true) Create an `IO`-like object for creating zeroed-out mmapped-memory that is not tied to a file -for use in [`Mmap.mmap`](@ref Mmap.mmap). Used by `SharedArray` for creating shared memory arrays. +for use in [`mmap`](@ref mmap). Used by `SharedArray` for creating shared memory arrays. # Examples ```jldoctest @@ -123,8 +125,8 @@ end # os-test # core implementation of mmap """ - Mmap.mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) - Mmap.mmap(type::Type{Array{T,N}}, dims) + mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) + mmap(type::Type{Array{T,N}}, dims) Create an `Array` whose values are linked to a file, using memory-mapping. This provides a convenient way of working with data too large to fit in the computer's memory. @@ -172,7 +174,7 @@ close(s) s = open("/tmp/mmap.bin") # default is read-only m = read(s, Int) n = read(s, Int) -A2 = Mmap.mmap(s, Matrix{Int}, (m,n)) +A2 = mmap(s, Matrix{Int}, (m,n)) ``` creates a `m`-by-`n` `Matrix{Int}`, linked to the file associated with stream `s`. @@ -254,10 +256,10 @@ mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true) where {T<:Array,N} = mmap(::Type{T}, i::Integer...; shared::Bool=true) where {T<:Array} = mmap(Anonymous(), T, convert(Tuple{Vararg{Int}},i), Int64(0); shared=shared) """ - Mmap.mmap(io, BitArray, [dims, offset]) + mmap(io, BitArray, [dims, offset]) Create a [`BitArray`](@ref) whose values are linked to a file, using memory-mapping; it has the same -purpose, works in the same way, and has the same arguments, as [`mmap`](@ref Mmap.mmap), but +purpose, works in the same way, and has the same arguments, as [`mmap`](@ref mmap), but the byte representation is different. # Examples @@ -266,7 +268,7 @@ julia> using Mmap julia> io = open("mmap.bin", "w+"); -julia> B = Mmap.mmap(io, BitArray, (25,30000)); +julia> B = mmap(io, BitArray, (25,30000)); julia> B[3, 4000] = true; @@ -276,7 +278,7 @@ julia> close(io); julia> io = open("mmap.bin", "r+"); -julia> C = Mmap.mmap(io, BitArray, (25,30000)); +julia> C = mmap(io, BitArray, (25,30000)); julia> C[3, 4000] true diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index ae99b48faad59c..51bf898e94b48f 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -5,53 +5,53 @@ using Test, Mmap, Random file = tempname() write(file, "Hello World\n") t = b"Hello World" -@test Mmap.mmap(file, Array{UInt8,3}, (11,1,1)) == reshape(t,(11,1,1)) +@test mmap(file, Array{UInt8,3}, (11,1,1)) == reshape(t,(11,1,1)) GC.gc(); GC.gc() -@test Mmap.mmap(file, Array{UInt8,3}, (1,11,1)) == reshape(t,(1,11,1)) +@test mmap(file, Array{UInt8,3}, (1,11,1)) == reshape(t,(1,11,1)) GC.gc(); GC.gc() -@test Mmap.mmap(file, Array{UInt8,3}, (1,1,11)) == reshape(t,(1,1,11)) +@test mmap(file, Array{UInt8,3}, (1,1,11)) == reshape(t,(1,1,11)) GC.gc(); GC.gc() -@test Mmap.mmap(file, Array{UInt8,3}, (11,0,1)) == Array{UInt8}(undef, (0,0,0)) -@test Mmap.mmap(file, Vector{UInt8}, (11,)) == t +@test mmap(file, Array{UInt8,3}, (11,0,1)) == Array{UInt8}(undef, (0,0,0)) +@test mmap(file, Vector{UInt8}, (11,)) == t GC.gc(); GC.gc() -@test Mmap.mmap(file, Array{UInt8,2}, (1,11)) == t' +@test mmap(file, Array{UInt8,2}, (1,11)) == t' GC.gc(); GC.gc() -@test Mmap.mmap(file, Array{UInt8,2}, (0,12)) == Array{UInt8}(undef, (0,0)) -m = Mmap.mmap(file, Array{UInt8,3}, (1,2,1)) +@test mmap(file, Array{UInt8,2}, (0,12)) == Array{UInt8}(undef, (0,0)) +m = mmap(file, Array{UInt8,3}, (1,2,1)) @test m == reshape(b"He",(1,2,1)) finalize(m); m=nothing; GC.gc() # constructors -@test length(@inferred Mmap.mmap(file)) == 12 -@test length(@inferred Mmap.mmap(file, Vector{Int8})) == 12 -@test length(@inferred Mmap.mmap(file, Matrix{Int8}, (12,1))) == 12 -@test length(@inferred Mmap.mmap(file, Matrix{Int8}, (12,1), 0)) == 12 -@test length(@inferred Mmap.mmap(file, Matrix{Int8}, (12,1), 0; grow=false)) == 12 -@test length(@inferred Mmap.mmap(file, Matrix{Int8}, (12,1), 0; shared=false)) == 12 -@test length(@inferred Mmap.mmap(file, Vector{Int8}, 12)) == 12 -@test length(@inferred Mmap.mmap(file, Vector{Int8}, 12, 0)) == 12 -@test length(@inferred Mmap.mmap(file, Vector{Int8}, 12, 0; grow=false)) == 12 -@test length(@inferred Mmap.mmap(file, Vector{Int8}, 12, 0; shared=false)) == 12 +@test length(@inferred mmap(file)) == 12 +@test length(@inferred mmap(file, Vector{Int8})) == 12 +@test length(@inferred mmap(file, Matrix{Int8}, (12,1))) == 12 +@test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0)) == 12 +@test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; grow=false)) == 12 +@test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; shared=false)) == 12 +@test length(@inferred mmap(file, Vector{Int8}, 12)) == 12 +@test length(@inferred mmap(file, Vector{Int8}, 12, 0)) == 12 +@test length(@inferred mmap(file, Vector{Int8}, 12, 0; grow=false)) == 12 +@test length(@inferred mmap(file, Vector{Int8}, 12, 0; shared=false)) == 12 s = open(file) -@test length(@inferred Mmap.mmap(s)) == 12 -@test length(@inferred Mmap.mmap(s, Vector{Int8})) == 12 -@test length(@inferred Mmap.mmap(s, Matrix{Int8}, (12,1))) == 12 -@test length(@inferred Mmap.mmap(s, Matrix{Int8}, (12,1), 0)) == 12 -@test length(@inferred Mmap.mmap(s, Matrix{Int8}, (12,1), 0; grow=false)) == 12 -@test length(@inferred Mmap.mmap(s, Matrix{Int8}, (12,1), 0; shared=false)) == 12 -@test length(@inferred Mmap.mmap(s, Vector{Int8}, 12)) == 12 -@test length(@inferred Mmap.mmap(s, Vector{Int8}, 12, 0)) == 12 -@test length(@inferred Mmap.mmap(s, Vector{Int8}, 12, 0; grow=false)) == 12 -@test length(@inferred Mmap.mmap(s, Vector{Int8}, 12, 0; shared=false)) == 12 +@test length(@inferred mmap(s)) == 12 +@test length(@inferred mmap(s, Vector{Int8})) == 12 +@test length(@inferred mmap(s, Matrix{Int8}, (12,1))) == 12 +@test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0)) == 12 +@test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; grow=false)) == 12 +@test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; shared=false)) == 12 +@test length(@inferred mmap(s, Vector{Int8}, 12)) == 12 +@test length(@inferred mmap(s, Vector{Int8}, 12, 0)) == 12 +@test length(@inferred mmap(s, Vector{Int8}, 12, 0; grow=false)) == 12 +@test length(@inferred mmap(s, Vector{Int8}, 12, 0; shared=false)) == 12 close(s) -@test_throws ErrorException Mmap.mmap(file, Vector{Ref}) # must be bit-type +@test_throws ErrorException mmap(file, Vector{Ref}) # must be bit-type GC.gc(); GC.gc() s = open(f->f,file,"w") -@test Mmap.mmap(file) == Vector{UInt8}() # requested len=0 on empty file -@test Mmap.mmap(file,Vector{UInt8},0) == Vector{UInt8}() +@test mmap(file) == Vector{UInt8}() # requested len=0 on empty file +@test mmap(file,Vector{UInt8},0) == Vector{UInt8}() s = open(file, "r+") -m = Mmap.mmap(s,Vector{UInt8},12) +m = mmap(s,Vector{UInt8},12) m[:] = b"Hello World\n" Mmap.sync!(m) close(s); finalize(m); m=nothing; GC.gc() @@ -59,36 +59,36 @@ close(s); finalize(m); m=nothing; GC.gc() s = open(file, "r") close(s) -@test_throws Base.IOError Mmap.mmap(s) # closed IOStream -@test_throws ArgumentError Mmap.mmap(s,Vector{UInt8},12,0) # closed IOStream -@test_throws SystemError Mmap.mmap("") +@test_throws Base.IOError mmap(s) # closed IOStream +@test_throws ArgumentError mmap(s,Vector{UInt8},12,0) # closed IOStream +@test_throws SystemError mmap("") # negative length -@test_throws ArgumentError Mmap.mmap(file, Vector{UInt8}, -1) +@test_throws ArgumentError mmap(file, Vector{UInt8}, -1) # negative offset -@test_throws ArgumentError Mmap.mmap(file, Vector{UInt8}, 1, -1) +@test_throws ArgumentError mmap(file, Vector{UInt8}, 1, -1) for i = 0x01:0x0c - @test length(Mmap.mmap(file, Vector{UInt8}, i)) == Int(i) + @test length(mmap(file, Vector{UInt8}, i)) == Int(i) end GC.gc(); GC.gc() sz = filesize(file) s = open(file, "r+") -m = Mmap.mmap(s, Vector{UInt8}, sz+1) +m = mmap(s, Vector{UInt8}, sz+1) @test length(m) == sz+1 # test growing @test m[end] == 0x00 close(s); finalize(m); m=nothing; GC.gc() sz = filesize(file) s = open(file, "r+") -m = Mmap.mmap(s, Vector{UInt8}, 1, sz) +m = mmap(s, Vector{UInt8}, 1, sz) @test length(m) == 1 @test m[1] == 0x00 close(s); finalize(m); m=nothing; GC.gc() sz = filesize(file) # test where offset is actually > than size of file; file is grown with zeroed bytes s = open(file, "r+") -m = Mmap.mmap(s, Vector{UInt8}, 1, sz+1) +m = mmap(s, Vector{UInt8}, 1, sz+1) @test length(m) == 1 @test m[1] == 0x00 close(s); finalize(m); m=nothing; GC.gc() @@ -98,7 +98,7 @@ close(s); finalize(m); m=nothing; GC.gc() # can thus not turn the segmentation fault into an exception. if !(Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le) s = open(file, "r") - m = Mmap.mmap(s) + m = mmap(s) @test_throws ReadOnlyMemoryError m[5] = UInt8('x') # tries to setindex! on read-only array finalize(m); m=nothing; GC.gc() end @@ -106,13 +106,13 @@ end write(file, "Hello World\n") s = open(file, "r") -m = Mmap.mmap(s) +m = mmap(s) close(s) finalize(m); m=nothing; GC.gc() -m = Mmap.mmap(file) +m = mmap(file) s = open(file, "r+") -c = Mmap.mmap(s) -d = Mmap.mmap(s) +c = mmap(s) +d = mmap(s) c[1] = UInt8('J') Mmap.sync!(c) close(s) @@ -125,18 +125,18 @@ write(file, "Hello World\n") s = open(file, "r") @test isreadonly(s) == true -c = Mmap.mmap(s, Vector{UInt8}, (11,)) +c = mmap(s, Vector{UInt8}, (11,)) @test c == b"Hello World" finalize(c); c=nothing; GC.gc() -c = Mmap.mmap(s, Vector{UInt8}, (UInt16(11),)) +c = mmap(s, Vector{UInt8}, (UInt16(11),)) @test c == b"Hello World" finalize(c); c=nothing; GC.gc() -@test_throws ArgumentError Mmap.mmap(s, Vector{UInt8}, (Int16(-11),)) -@test_throws ArgumentError Mmap.mmap(s, Vector{UInt8}, (typemax(UInt),)) +@test_throws ArgumentError mmap(s, Vector{UInt8}, (Int16(-11),)) +@test_throws ArgumentError mmap(s, Vector{UInt8}, (typemax(UInt),)) close(s) s = open(file, "r+") @test isreadonly(s) == false -c = Mmap.mmap(s, Vector{UInt8}, (11,)) +c = mmap(s, Vector{UInt8}, (11,)) c[5] = UInt8('x') Mmap.sync!(c) close(s) @@ -146,18 +146,18 @@ close(s) @test startswith(str, "Hellx World") finalize(c); c=nothing; GC.gc() -c = Mmap.mmap(file) +c = mmap(file) @test c == b"Hellx World\n" finalize(c); c=nothing; GC.gc() -c = Mmap.mmap(file, Vector{UInt8}, 3) +c = mmap(file, Vector{UInt8}, 3) @test c == b"Hel" finalize(c); c=nothing; GC.gc() s = open(file, "r") -c = Mmap.mmap(s, Vector{UInt8}, 6) +c = mmap(s, Vector{UInt8}, 6) @test c == b"Hellx " close(s) finalize(c); c=nothing; GC.gc() -c = Mmap.mmap(file, Vector{UInt8}, 5, 6) +c = mmap(file, Vector{UInt8}, 5, 6) @test c == b"World" finalize(c); c=nothing; GC.gc() @@ -165,8 +165,8 @@ s = open(file, "w") write(s, "Hello World\n") close(s) -# test Mmap.mmap -m = Mmap.mmap(file) +# test mmap +m = mmap(file) tdata = b"Hello World\n" for i = 1:12 @test m[i] == tdata[i] @@ -174,7 +174,7 @@ end @test_throws BoundsError m[13] finalize(m); m=nothing; GC.gc() -m = Mmap.mmap(file,Vector{UInt8},6) +m = mmap(file,Vector{UInt8},6) @test m[1] == b"H"[1] @test m[2] == b"e"[1] @test m[3] == b"l"[1] @@ -184,7 +184,7 @@ m = Mmap.mmap(file,Vector{UInt8},6) @test_throws BoundsError m[7] finalize(m); m=nothing; GC.gc() -m = Mmap.mmap(file,Vector{UInt8},2,6) +m = mmap(file,Vector{UInt8},2,6) @test m[1] == b"W"[1] @test m[2] == b"o"[1] @test_throws BoundsError m[3] @@ -198,13 +198,13 @@ write(s, [0xffffffffffffffff, close(s) s = open(file, "r") @test isreadonly(s) -b = @inferred Mmap.mmap(s, BitArray, (17,13)) +b = @inferred mmap(s, BitArray, (17,13)) @test Test._check_bitarray_consistency(b) @test b == trues(17,13) -@test_throws ArgumentError Mmap.mmap(s, BitArray, (7,3)) +@test_throws ArgumentError mmap(s, BitArray, (7,3)) close(s) s = open(file, "r+") -b = Mmap.mmap(s, BitArray, (17,19)) +b = mmap(s, BitArray, (17,19)) @test Test._check_bitarray_consistency(b) rand!(b) Mmap.sync!(b) @@ -213,7 +213,7 @@ b0 = copy(b) close(s) s = open(file, "r") @test isreadonly(s) -b = Mmap.mmap(s, BitArray, (17,19)) +b = mmap(s, BitArray, (17,19)) @test Test._check_bitarray_consistency(b) @test b == b0 close(s) @@ -227,29 +227,29 @@ open(file,"w") do f end @test filesize(file) == 9 s = open(file, "r+") -m = Mmap.mmap(s, BitArray, (72,)) +m = mmap(s, BitArray, (72,)) @test Test._check_bitarray_consistency(m) @test length(m) == 72 close(s); finalize(m); m = nothing; GC.gc() -m = Mmap.mmap(file, BitArray, (72,)) +m = mmap(file, BitArray, (72,)) @test Test._check_bitarray_consistency(m) @test length(m) == 72 finalize(m); m = nothing; GC.gc() s = open(file, "r+") -m = Mmap.mmap(s, BitArray, 72) # len integer instead of dims +m = mmap(s, BitArray, 72) # len integer instead of dims @test Test._check_bitarray_consistency(m) @test length(m) == 72 close(s); finalize(m); m = nothing; GC.gc() -m = Mmap.mmap(file, BitArray, 72) # len integer instead of dims +m = mmap(file, BitArray, 72) # len integer instead of dims @test Test._check_bitarray_consistency(m) @test length(m) == 72 finalize(m); m = nothing; GC.gc() rm(file) -# Mmap.mmap with an offset +# mmap with an offset A = rand(1:20, 500, 300) fname = tempname() s = open(fname, "w+") @@ -260,12 +260,12 @@ close(s) s = open(fname) m = read(s, Int) n = read(s, Int) -A2 = Mmap.mmap(s, Matrix{Int}, (m,n)) +A2 = mmap(s, Matrix{Int}, (m,n)) @test A == A2 seek(s, 0) -A3 = Mmap.mmap(s, Matrix{Int}, (m,n), convert(Int64, 2*sizeof(Int))) +A3 = mmap(s, Matrix{Int}, (m,n), convert(Int64, 2*sizeof(Int))) @test A == A3 -A4 = Mmap.mmap(s, Matrix{Int}, (m,150), convert(Int64, (2+150*m)*sizeof(Int))) +A4 = mmap(s, Matrix{Int}, (m,150), convert(Int64, (2+150*m)*sizeof(Int))) @test A[:, 151:end] == A4 close(s) finalize(A2); finalize(A3); finalize(A4) @@ -282,7 +282,7 @@ m = Mmap.Anonymous() @test isreadable(m) @test iswritable(m) -m = Mmap.mmap(Vector{UInt8}, 12) +m = mmap(Vector{UInt8}, 12) @test length(m) == 12 @test all(m .== 0x00) @test m[1] === 0x00 @@ -290,16 +290,16 @@ m = Mmap.mmap(Vector{UInt8}, 12) m[1] = 0x0a Mmap.sync!(m) @test m[1] === 0x0a -m = Mmap.mmap(Vector{UInt8}, 12; shared=false) -m = Mmap.mmap(Vector{Int}, 12) +m = mmap(Vector{UInt8}, 12; shared=false) +m = mmap(Vector{Int}, 12) @test length(m) == 12 @test all(m .== 0) @test m[1] === 0 @test m[end] === 0 -m = Mmap.mmap(Vector{Float64}, 12) +m = mmap(Vector{Float64}, 12) @test length(m) == 12 @test all(m .== 0.0) -m = Mmap.mmap(Matrix{Int8}, (12,12)) +m = mmap(Matrix{Int8}, (12,12)) @test size(m) == (12,12) @test all(m == zeros(Int8, (12,12))) @test sizeof(m) == prod((12,12)) @@ -315,12 +315,12 @@ finalize(m); m = nothing; GC.gc() if Sys.isunix() file = tempname() write(file, rand(Float64, 20)) - A = Mmap.mmap(file, Vector{Float64}, 20) + A = mmap(file, Vector{Float64}, 20) @test Mmap.madvise!(A, Mmap.MADV_WILLNEED) === nothing # checking for no error finalize(A); A = nothing; GC.gc() write(file, BitArray(rand(Bool, 20))) - b = Mmap.mmap(file, BitArray, 20) + b = mmap(file, BitArray, 20) @test Mmap.madvise!(b, Mmap.MADV_WILLNEED) === nothing finalize(b); b = nothing; GC.gc() rm(file) @@ -330,10 +330,10 @@ end file = tempname() touch(file) open(file, "r+") do s - A = Mmap.mmap(s, Vector{UInt8}, (10,), 0) + A = mmap(s, Vector{UInt8}, (10,), 0) Mmap.sync!(A) finalize(A); A = nothing; GC.gc() - A = Mmap.mmap(s, Vector{UInt8}, (10,), 1) + A = mmap(s, Vector{UInt8}, (10,), 1) Mmap.sync!(A) finalize(A); A = nothing; GC.gc() end diff --git a/stdlib/SharedArrays/src/SharedArrays.jl b/stdlib/SharedArrays/src/SharedArrays.jl index 08a7b6fbe2b3e4..347d22180f7b5c 100644 --- a/stdlib/SharedArrays/src/SharedArrays.jl +++ b/stdlib/SharedArrays/src/SharedArrays.jl @@ -206,7 +206,7 @@ function SharedArray{T,N}(filename::AbstractString, dims::NTuple{N,Int}, offset: # Create the file if it doesn't exist, map it if it does refs = Vector{Future}(undef, length(pids)) func_mmap = mode -> open(filename, mode) do io - Mmap.mmap(io, Array{T,N}, dims, offset; shared=true) + mmap(io, Array{T,N}, dims, offset; shared=true) end s = Array{T}(undef, ntuple(d->0,N)) if onlocalhost @@ -668,7 +668,7 @@ function _shm_mmap_array(T, dims, shm_seg_name, mode) readonly = !((mode & JL_O_RDWR) == JL_O_RDWR) create = (mode & JL_O_CREAT) == JL_O_CREAT s = Mmap.Anonymous(shm_seg_name, readonly, create) - Mmap.mmap(s, Array{T,length(dims)}, dims, zero(Int64)) + mmap(s, Array{T,length(dims)}, dims, zero(Int64)) end # no-op in windows @@ -688,7 +688,7 @@ function _shm_mmap_array(T, dims, shm_seg_name, mode) systemerror("ftruncate() failed for shm segment " * shm_seg_name, rc != 0) end - Mmap.mmap(s, Array{T,length(dims)}, dims, zero(Int64); grow=false) + mmap(s, Array{T,length(dims)}, dims, zero(Int64); grow=false) end shm_unlink(shm_seg_name) = ccall(:shm_unlink, Cint, (Cstring,), shm_seg_name)