diff --git a/NEWS.md b/NEWS.md index 8907275925155..52180a852b0e7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -21,6 +21,7 @@ Build system changes New library functions --------------------- +* `copyuntil(out, io, delim)` and `copyline(out, io)` copy data into an `out::IO` stream ([#48273]). New library features -------------------- diff --git a/base/exports.jl b/base/exports.jl index 10f43825e12df..0959fa1c391e2 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -857,6 +857,8 @@ export readline, readlines, readuntil, + copyuntil, + copyline, redirect_stdio, redirect_stderr, redirect_stdin, diff --git a/base/io.jl b/base/io.jl index 60a24831587cb..c62d6393d12ec 100644 --- a/base/io.jl +++ b/base/io.jl @@ -440,10 +440,10 @@ for f in ( end read(io::AbstractPipe, byte::Type{UInt8}) = read(pipe_reader(io)::IO, byte)::UInt8 unsafe_read(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_read(pipe_reader(io)::IO, p, nb) -readuntil(io::AbstractPipe, arg::UInt8; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) -readuntil(io::AbstractPipe, arg::AbstractChar; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) -readuntil(io::AbstractPipe, arg::AbstractString; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) -readuntil(io::AbstractPipe, arg::AbstractVector; kw...) = readuntil(pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::UInt8; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::AbstractChar; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::AbstractString; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) +copyuntil(out::IO, io::AbstractPipe, arg::AbstractVector; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) readuntil_vector!(io::AbstractPipe, target::AbstractVector, keep::Bool, out) = readuntil_vector!(pipe_reader(io)::IO, target, keep, out) readbytes!(io::AbstractPipe, target::AbstractVector{UInt8}, n=length(target)) = readbytes!(pipe_reader(io)::IO, target, n) peek(io::AbstractPipe, ::Type{T}) where {T} = peek(pipe_reader(io)::IO, T)::T @@ -499,11 +499,15 @@ read!(filename::AbstractString, a) = open(io->read!(io, a), convert(String, file readuntil(stream::IO, delim; keep::Bool = false) readuntil(filename::AbstractString, delim; keep::Bool = false) -Read a string from an I/O stream or a file, up to the given delimiter. +Read a string from an I/O `stream` or a file, up to the given delimiter. The delimiter can be a `UInt8`, `AbstractChar`, string, or vector. Keyword argument `keep` controls whether the delimiter is included in the result. The text is assumed to be encoded in UTF-8. +Return a `String` if `delim` is an `AbstractChar` or a string +or otherwise return a `Vector{typeof(delim)}`. See also [`copyuntil`](@ref) +to instead write in-place to another stream (which can be a preallocated [`IOBuffer`](@ref)). + # Examples ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); @@ -517,7 +521,39 @@ julia> readuntil("my_file.txt", '.', keep = true) julia> rm("my_file.txt") ``` """ -readuntil(filename::AbstractString, args...; kw...) = open(io->readuntil(io, args...; kw...), convert(String, filename)::String) +readuntil(filename::AbstractString, delim; kw...) = open(io->readuntil(io, delim; kw...), convert(String, filename)::String) +readuntil(stream::IO, delim::UInt8; kw...) = _unsafe_take!(copyuntil(IOBuffer(sizehint=70), stream, delim; kw...)) +readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = String(_unsafe_take!(copyuntil(IOBuffer(sizehint=70), stream, delim; kw...))) + + +""" + copyuntil(out::IO, stream::IO, delim; keep::Bool = false) + copyuntil(out::IO, filename::AbstractString, delim; keep::Bool = false) + +Copy a string from an I/O `stream` or a file, up to the given delimiter, to +the `out` stream, returning `out`. +The delimiter can be a `UInt8`, `AbstractChar`, string, or vector. +Keyword argument `keep` controls whether the delimiter is included in the result. +The text is assumed to be encoded in UTF-8. + +Similar to [`readuntil`](@ref), which returns a `String`; in contrast, +`copyuntil` writes directly to `out`, without allocating a string. +(This can be used, for example, to read data into a pre-allocated [`IOBuffer`](@ref).) + +# Examples +```jldoctest +julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); + +julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", 'L'))) +"Julia" + +julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true))) +"JuliaLang is a GitHub organization." + +julia> rm("my_file.txt") +``` +""" +copyuntil(out::IO, filename::AbstractString, delim; kw...) = open(io->copyuntil(out, io, delim; kw...), convert(String, filename)::String) """ readline(io::IO=stdin; keep::Bool=false) @@ -530,6 +566,11 @@ false (as it is by default), these trailing newline characters are removed from line before it is returned. When `keep` is true, they are returned as part of the line. +Return a `String`. See also [`copyline`](@ref) to instead write in-place +to another stream (which can be a preallocated [`IOBuffer`](@ref)). + +See also [`readuntil`](@ref) for reading until more general delimiters. + # Examples ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); @@ -551,21 +592,63 @@ Logan "Logan" ``` """ -function readline(filename::AbstractString; keep::Bool=false) - open(filename) do f - readline(f, keep=keep) - end -end +readline(filename::AbstractString; keep::Bool=false) = + open(io -> readline(io; keep), filename) +readline(s::IO=stdin; keep::Bool=false) = + String(_unsafe_take!(copyline(IOBuffer(sizehint=70), s; keep))) -function readline(s::IO=stdin; keep::Bool=false) - line = readuntil(s, 0x0a, keep=true)::Vector{UInt8} - i = length(line) - if keep || i == 0 || line[i] != 0x0a - return String(line) - elseif i < 2 || line[i-1] != 0x0d - return String(resize!(line,i-1)) +""" + copyline(out::IO, io::IO=stdin; keep::Bool=false) + copyline(out::IO, filename::AbstractString; keep::Bool=false) + +Copy a single line of text from an I/O `stream` or a file to the `out` stream, +returning `out`. + +When reading from a file, the text is assumed to be encoded in UTF-8. Lines in the +input end with `'\\n'` or `"\\r\\n"` or the end of an input stream. When `keep` is +false (as it is by default), these trailing newline characters are removed from the +line before it is returned. When `keep` is true, they are returned as part of the +line. + +Similar to [`readline`](@ref), which returns a `String`; in contrast, +`copyline` writes directly to `out`, without allocating a string. +(This can be used, for example, to read data into a pre-allocated [`IOBuffer`](@ref).) + +See also [`copyuntil`](@ref) for reading until more general delimiters. + +# Examples +```jldoctest +julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); + +julia> String(take!(copyline(IOBuffer(), "my_file.txt"))) +"JuliaLang is a GitHub organization." + +julia> String(take!(copyline(IOBuffer(), "my_file.txt", keep=true))) +"JuliaLang is a GitHub organization.\\n" + +julia> rm("my_file.txt") +``` +""" +copyline(out::IO, filename::AbstractString; keep::Bool=false) = + open(io -> copyline(out, io; keep), filename) + +# fallback to optimized methods for IOBuffer in iobuffer.jl +function copyline(out::IO, s::IO; keep::Bool=false) + if keep + return copyuntil(out, s, 0x0a, keep=true) else - return String(resize!(line,i-2)) + # more complicated to deal with CRLF logic + while !eof(s) + b = read(s, UInt8) + b == 0x0a && break + if b == 0x0d && !eof(s) + b = read(s, UInt8) + b == 0x0a && break + write(out, 0x0d) + end + write(out, b) + end + return out end end @@ -816,15 +899,10 @@ end # read(io, T) is not defined for other AbstractChar: implementations # must provide their own encoding-specific method. -# readuntil_string is useful below since it has -# an optimized method for s::IOStream -readuntil_string(s::IO, delim::UInt8, keep::Bool) = String(readuntil(s, delim, keep=keep))::String - -function readuntil(s::IO, delim::AbstractChar; keep::Bool=false) +function copyuntil(out::IO, s::IO, delim::AbstractChar; keep::Bool=false) if delim ≤ '\x7f' - return readuntil_string(s, delim % UInt8, keep) + return copyuntil(out, s, delim % UInt8; keep) end - out = IOBuffer() for c in readeach(s, Char) if c == delim keep && write(out, c) @@ -832,20 +910,27 @@ function readuntil(s::IO, delim::AbstractChar; keep::Bool=false) end write(out, c) end - return String(take!(out)) + return out end -function readuntil(s::IO, delim::T; keep::Bool=false) where T - out = (T === UInt8 ? StringVector(0) : Vector{T}()) +# note: optimized methods of copyuntil for IOStreams and delim::UInt8 in iostream.jl +function _copyuntil(out, s::IO, delim::T, keep::Bool) where T + output! = isa(out, IO) ? write : push! for c in readeach(s, T) if c == delim - keep && push!(out, c) + keep && output!(out, c) break end - push!(out, c) + output!(out, c) end return out end +readuntil(s::IO, delim::T; keep::Bool=false) where T = + _copyuntil(Vector{T}(), s, delim, keep) +readuntil(s::IO, delim::UInt8; keep::Bool=false) = + _copyuntil(resize!(StringVector(70), 0), s, delim, keep) +copyuntil(out::IO, s::IO, delim::T; keep::Bool=false) where T = + _copyuntil(out, s, delim, keep) # requires that indices for target are the integer unit range from firstindex to lastindex # returns whether the delimiter was matched @@ -933,27 +1018,29 @@ function readuntil_vector!(io::IO, target::AbstractVector{T}, keep::Bool, out) w return false end -function readuntil(io::IO, target::AbstractString; keep::Bool=false) +function copyuntil(out::IO, io::IO, target::AbstractString; keep::Bool=false) # small-string target optimizations x = Iterators.peel(target) - isnothing(x) && return "" + isnothing(x) && return out c, rest = x if isempty(rest) && c <= '\x7f' - return readuntil_string(io, c % UInt8, keep) + return copyuntil(out, io, c % UInt8; keep) end # convert String to a utf8-byte-iterator if !(target isa String) && !(target isa SubString{String}) target = String(target) end target = codeunits(target)::AbstractVector - return String(readuntil(io, target, keep=keep)) + return copyuntil(out, io, target, keep=keep) end function readuntil(io::IO, target::AbstractVector{T}; keep::Bool=false) where T - out = (T === UInt8 ? StringVector(0) : Vector{T}()) + out = (T === UInt8 ? resize!(StringVector(70), 0) : Vector{T}()) readuntil_vector!(io, target, keep, out) return out end +copyuntil(out::IO, io::IO, target::AbstractVector; keep::Bool=false) = + (readuntil_vector!(io, target, keep, out); out) """ readchomp(x) @@ -1128,7 +1215,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) empty!(chunks) # will cause next iteration to terminate seekend(r.itr.stream) # reposition to end of stream for isdone - s = String(take!(buf)) + s = String(_unsafe_take!(buf)) else # extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline] if ichunk == jchunk # common case: current and previous newline in same chunk @@ -1145,7 +1232,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) end write(buf, view(chunks[jchunk], 1:jnewline)) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) - s = String(take!(buf)) + s = String(_unsafe_take!(buf)) # overwrite obsolete chunks (ichunk+1:jchunk) i = jchunk diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 6c95285f232f2..deb86e774f4e4 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -516,33 +516,53 @@ function occursin(delim::UInt8, buf::GenericIOBuffer) return false end -function readuntil(io::GenericIOBuffer, delim::UInt8; keep::Bool=false) - lb = 70 - A = StringVector(lb) - nread = 0 - nout = 0 - data = io.data - for i = io.ptr : io.size - @inbounds b = data[i] - nread += 1 - if keep || b != delim - nout += 1 - if nout > lb - lb = nout*2 - resize!(A, lb) - end - @inbounds A[nout] = b - end - if b == delim - break - end +function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) + data = view(io.data, io.ptr:io.size) + # note: findfirst + copyto! is much faster than a single loop + # except for nout ≲ 20. A single loop is 2x faster for nout=5. + nout = nread = something(findfirst(==(delim), data), length(data)) + if !keep && nout > 0 && data[nout] == delim + nout -= 1 end + write(out, view(io.data, io.ptr:io.ptr+nout-1)) io.ptr += nread - if lb != nout - resize!(A, nout) + return out +end + +function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) + copyuntil(out, s, 0x0a, keep=true) + line = out.data + i = out.size + if keep || i == 0 || line[i] != 0x0a + return out + elseif i < 2 || line[i-1] != 0x0d + i -= 1 + else + i -= 2 end - A + out.size = i + if !out.append + out.ptr = i+1 + end + return out +end + +function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) + data = view(io.data, io.ptr:io.size) + # note: findfirst + copyto! is much faster than a single loop + # except for nout ≲ 20. A single loop is 2x faster for nout=5. + nout = nread = something(findfirst(==(0x0a), data), length(data)) + if !keep && nout > 0 && data[nout] == 0x0a + nout -= 1 + nout > 0 && data[nout] == 0x0d && (nout -= 1) + end + write(out, view(io.data, io.ptr:io.ptr+nout-1)) + io.ptr += nread + return out end +copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) +copyline(out::GenericIOBuffer, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) + # copy-free crc32c of IOBuffer: function _crc32c(io::IOBuffer, nb::Integer, crc::UInt32=0x00000000) diff --git a/base/iostream.jl b/base/iostream.jl index 23dfb53256e82..f5a8c0a8dffc8 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -443,11 +443,46 @@ end function readuntil_string(s::IOStream, delim::UInt8, keep::Bool) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 1, !keep) end +readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false) = + delim ≤ '\x7f' ? readuntil_string(s, delim % UInt8, keep) : + String(unsafe_take!(copyuntil(IOBuffer(sizehint=70), s, delim; keep))) function readline(s::IOStream; keep::Bool=false) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2) end +function copyuntil(out::IOBuffer, s::IOStream, delim::UInt8; keep::Bool=false) + ensureroom(out, 16) + ptr = (out.append ? out.size+1 : out.ptr) + d = out.data + len = length(d) + while true + GC.@preserve d @_lock_ios s n= + Int(ccall(:jl_readuntil_buf, Csize_t, (Ptr{Cvoid}, UInt8, Ptr{UInt8}, Csize_t), + s.ios, delim, pointer(d, ptr), (len - ptr + 1) % Csize_t)) + iszero(n) && break + ptr += n + if d[ptr-1] == delim + keep || (ptr -= 1) + break + end + (eof(s) || len == out.maxsize) && break + len = min(2len + 64, out.maxsize) + resize!(d, len) + end + out.size = max(out.size, ptr - 1) + if !out.append + out.ptr = ptr + end + return out +end + +function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) + @_lock_ios out @_lock_ios s ccall(:ios_copyuntil, Csize_t, + (Ptr{Cvoid}, Ptr{Cvoid}, UInt8, Cint), out.ios, s.ios, delim, keep) + return out +end + function readbytes_all!(s::IOStream, b::Union{Array{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, nb::Integer) diff --git a/doc/src/base/io-network.md b/doc/src/base/io-network.md index 4e371039f1a9b..68f144427a892 100644 --- a/doc/src/base/io-network.md +++ b/doc/src/base/io-network.md @@ -71,6 +71,8 @@ Base.readline Base.readuntil Base.readlines Base.eachline +Base.copyline +Base.copyuntil Base.displaysize ``` diff --git a/src/flisp/iostream.c b/src/flisp/iostream.c index b2b2477bb43c6..c1c6d965d2917 100644 --- a/src/flisp/iostream.c +++ b/src/flisp/iostream.c @@ -354,7 +354,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_setbuf(&dest, data, 80, 0); char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil"); ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil"); - size_t n = ios_copyuntil(&dest, src, delim); + size_t n = ios_copyuntil(&dest, src, delim, 1); cv->len = n; if (dest.buf != data) { // outgrew initial space @@ -376,7 +376,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil"); ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil"); char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil"); - return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim)); + return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 1)); } value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) diff --git a/src/support/ios.c b/src/support/ios.c index b5a168f705603..c98c529991642 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -832,7 +832,7 @@ size_t ios_copyall(ios_t *to, ios_t *from) #define LINE_CHUNK_SIZE 160 -size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, int keep) { size_t total = 0, avail = (size_t)(from->size - from->bpos); while (!ios_eof(from)) { @@ -850,9 +850,9 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) avail = 0; } else { - size_t ntowrite = pd - (from->buf+from->bpos) + 1; + size_t ntowrite = pd - (from->buf+from->bpos) + (keep != 0); written = ios_write(to, from->buf+from->bpos, ntowrite); - from->bpos += ntowrite; + from->bpos += ntowrite + (keep == 0); total += written; return total; } @@ -1217,7 +1217,7 @@ char *ios_readline(ios_t *s) { ios_t dest; ios_mem(&dest, 0); - ios_copyuntil(&dest, s, '\n'); + ios_copyuntil(&dest, s, '\n', 1); size_t n; return ios_take_buffer(&dest, &n); } diff --git a/src/support/ios.h b/src/support/ios.h index 2547555b5585d..6eab9e21c45b6 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -108,7 +108,7 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s); JL_DLLEXPORT void ios_set_readonly(ios_t *s); JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes); JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from); -JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) JL_NOTSAFEPOINT; +JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, int keep) JL_NOTSAFEPOINT; JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite); // ensure at least n bytes are buffered if possible. returns # available. JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n); diff --git a/src/sys.c b/src/sys.c index 2de4bc61a20b8..d55c5df7ab066 100644 --- a/src/sys.c +++ b/src/sys.c @@ -288,7 +288,7 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint ios_t dest; ios_mem(&dest, 0); ios_setbuf(&dest, (char*)a->data, 80, 0); - size_t n = ios_copyuntil(&dest, s, delim); + size_t n = ios_copyuntil(&dest, s, delim, 1); if (chomp && n > 0 && dest.buf[n - 1] == delim) { n--; if (chomp == 2 && n > 0 && dest.buf[n - 1] == '\r') { @@ -316,6 +316,50 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint return (jl_value_t*)a; } +// read up to buflen bytes, including delim, into buf. returns number of bytes read. +JL_DLLEXPORT size_t jl_readuntil_buf(ios_t *s, uint8_t delim, uint8_t *buf, size_t buflen) +{ + // manually inlined common case + size_t avail = (size_t)(s->size - s->bpos); + if (avail > buflen) avail = buflen; + char *pd = (char*)memchr(s->buf + s->bpos, delim, avail); + if (pd) { + size_t n = pd - (s->buf + s->bpos) + 1; + memcpy(buf, s->buf + s->bpos, n); + s->bpos += n; + return n; + } + else { + size_t total = avail; + memcpy(buf, s->buf + s->bpos, avail); + s->bpos += avail; + if (avail == buflen) return total; + + // code derived from ios_copyuntil + while (!ios_eof(s)) { + avail = ios_readprep(s, 160); // read LINE_CHUNK_SIZE + if (avail == 0) break; + if (total+avail > buflen) avail = buflen-total; + char *pd = (char*)memchr(s->buf+s->bpos, delim, avail); + if (pd == NULL) { + memcpy(buf+total, s->buf+s->bpos, avail); + s->bpos += avail; + total += avail; + if (buflen == total) return total; + } + else { + size_t ntowrite = pd - (s->buf+s->bpos) + 1; + memcpy(buf+total, s->buf+s->bpos, ntowrite); + s->bpos += ntowrite; + total += ntowrite; + return total; + } + } + s->_eof = 1; + return total; + } +} + JL_DLLEXPORT int jl_ios_buffer_n(ios_t *s, const size_t n) { size_t space, ret; diff --git a/test/read.jl b/test/read.jl index b8060a023333f..ff0952b1495da 100644 --- a/test/read.jl +++ b/test/read.jl @@ -145,6 +145,7 @@ for (name, f) in l verbose && println("$name readuntil...") for (t, s, m, kept) in [ + ("a", "", "", ""), ("a", "ab", "a", "a"), ("b", "ab", "b", "b"), ("α", "αγ", "α", "α"), @@ -152,16 +153,19 @@ for (name, f) in l ("bc", "abc", "bc", "bc"), ("αβ", "αβγ", "αβ", "αβ"), ("aaabc", "ab", "aa", "aaab"), + ("aaabc", "b", "aaa", "aaab"), ("aaabc", "ac", "aaabc", "aaabc"), ("aaabc", "aab", "a", "aaab"), ("aaabc", "aac", "aaabc", "aaabc"), ("αααβγ", "αβ", "αα", "αααβ"), + ("αααβγ", "β", "ααα", "αααβ"), ("αααβγ", "ααβ", "α", "αααβ"), ("αααβγ", "αγ", "αααβγ", "αααβγ"), ("barbarbarians", "barbarian", "bar", "barbarbarian"), ("abcaabcaabcxl", "abcaabcx", "abca", "abcaabcaabcx"), ("abbaabbaabbabbaax", "abbaabbabbaax", "abba", "abbaabbaabbabbaax"), ("abbaabbabbaabbaabbabbaax", "abbaabbabbaax", "abbaabbabba", "abbaabbabbaabbaabbabbaax"), + ('a'^500 * 'x' * "bbbb", "x", 'a'^500, 'a'^500 * 'x'), ] local t, s, m, kept @test readuntil(io(t), s) == m @@ -174,6 +178,18 @@ for (name, f) in l @test readuntil(io(t), unsafe_wrap(Vector{UInt8},s), keep=true) == unsafe_wrap(Vector{UInt8},kept) @test readuntil(io(t), collect(s)::Vector{Char}) == Vector{Char}(m) @test readuntil(io(t), collect(s)::Vector{Char}, keep=true) == Vector{Char}(kept) + + buf = IOBuffer() + @test String(take!(copyuntil(buf, io(t), s))) == m + @test String(take!(copyuntil(buf, io(t), s, keep=true))) == kept + file = tempname() + for (k,m) in ((false, m), (true, kept)) + open(file, "w") do f + @test f == copyuntil(f, io(t), s, keep=k) + end + @test read(file, String) == m + end + rm(file) end cleanup() @@ -281,8 +297,45 @@ for (name, f) in l cleanup() verbose && println("$name readline...") - @test readline(io(), keep=true) == readline(IOBuffer(text), keep=true) - @test readline(io(), keep=true) == readline(filename, keep=true) + file = tempname() + for lineending in ("\n", "\r\n", "") + kept = "foo bar" * lineending + t = isempty(lineending) ? "foo bar" : kept * "baz\n" + write(file, t) + @test readline(io(t)) == readline(file) == "foo bar" + @test readline(io(t), keep=true) == readline(file, keep=true) == kept + + @test String(take!(copyline(IOBuffer(), file))) == "foo bar" + @test String(take!(copyline(IOBuffer(), file, keep=true))) == kept + + cleanup() + + buf = IOBuffer() + @test buf === copyline(buf, io(t)) + @test String(take!(buf)) == "foo bar" + @test String(take!(copyline(buf, file, keep=true))) == kept + for keep in (true, false) + open(file, "w") do f + @test f === copyline(f, io(t), keep=keep) + end + @test read(file, String) == (keep ? kept : "foo bar") + end + + cleanup() + + write(file, lineending) + @test readline(IOBuffer(lineending)) == "" + @test readline(IOBuffer(lineending), keep=true) == lineending + @test String(take!(copyline(IOBuffer(), IOBuffer(lineending)))) == "" + @test String(take!(copyline(IOBuffer(), IOBuffer(lineending), keep=true))) == lineending + @test readline(file) == "" + @test readline(file, keep=true) == lineending + @test String(take!(copyline(IOBuffer(), file))) == "" + @test String(take!(copyline(IOBuffer(), file, keep=true))) == lineending + + cleanup() + end + rm(file) verbose && println("$name readlines...") @test readlines(io(), keep=true) == readlines(IOBuffer(text), keep=true)