From 2e47cf1cdd64c0a3313d9639d6587d89e52a2647 Mon Sep 17 00:00:00 2001 From: Mikhail Tavarez Date: Fri, 3 May 2024 10:03:24 -0500 Subject: [PATCH] update for 24.3 --- README.md | 4 +- external/gojo/bufio/__init__.mojo | 2 + external/gojo/bufio/bufio.mojo | 938 +++++++++++++++++++++++ external/gojo/bufio/scan.mojo | 458 +++++++++++ external/gojo/bytes/__init__.mojo | 2 + external/gojo/bytes/buffer.mojo | 648 ++++++++++++++++ external/gojo/bytes/reader.mojo | 216 ++++++ external/gojo/fmt/__init__.mojo | 2 +- external/gojo/fmt/fmt.mojo | 136 +++- external/gojo/net/__init__.mojo | 4 + external/gojo/net/address.mojo | 145 ++++ external/gojo/net/dial.mojo | 45 ++ external/gojo/net/fd.mojo | 77 ++ external/gojo/net/ip.mojo | 178 +++++ external/gojo/net/net.mojo | 130 ++++ external/gojo/net/socket.mojo | 432 +++++++++++ external/gojo/net/tcp.mojo | 207 +++++ external/gojo/syscall/__init__.mojo | 0 external/gojo/syscall/file.mojo | 110 +++ external/gojo/syscall/net.mojo | 749 ++++++++++++++++++ external/gojo/syscall/types.mojo | 63 ++ external/gojo/unicode/__init__.mojo | 1 + external/gojo/unicode/utf8/__init__.mojo | 4 + external/gojo/unicode/utf8/runes.mojo | 353 +++++++++ external/hue/happy_palettegen.mojo | 2 +- external/mist/__init__.mojo | 2 +- external/mist/color.mojo | 4 +- external/mist/profile.mojo | 6 +- 28 files changed, 4881 insertions(+), 37 deletions(-) create mode 100644 external/gojo/bufio/__init__.mojo create mode 100644 external/gojo/bufio/bufio.mojo create mode 100644 external/gojo/bufio/scan.mojo create mode 100644 external/gojo/bytes/__init__.mojo create mode 100644 external/gojo/bytes/buffer.mojo create mode 100644 external/gojo/bytes/reader.mojo create mode 100644 external/gojo/net/__init__.mojo create mode 100644 external/gojo/net/address.mojo create mode 100644 external/gojo/net/dial.mojo create mode 100644 external/gojo/net/fd.mojo create mode 100644 external/gojo/net/ip.mojo create mode 100644 external/gojo/net/net.mojo create mode 100644 external/gojo/net/socket.mojo create mode 100644 external/gojo/net/tcp.mojo create mode 100644 external/gojo/syscall/__init__.mojo create mode 100644 external/gojo/syscall/file.mojo create mode 100644 external/gojo/syscall/net.mojo create mode 100644 external/gojo/syscall/types.mojo create mode 100644 external/gojo/unicode/__init__.mojo create mode 100644 external/gojo/unicode/utf8/__init__.mojo create mode 100644 external/gojo/unicode/utf8/runes.mojo diff --git a/README.md b/README.md index 5a04a30..35d905b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # stump -WIP Logger! Inspired by charmbracelet's log package and the Python structlog package. +![Mojo 24.3](https://img.shields.io/badge/Mojo%F0%9F%94%A5-24.3-purple) -NOTE: This does not work on Mojo 24.2, you must use the nightly build for now. This will be resolved in the next Mojo release. +WIP Logger! Inspired by charmbracelet's log package and the Python structlog package. There are some things I'm ironing out around terminal color profile querying at compilation time. At the moment, the default styles assume a `TRUE_COLOR` enabled color profile. So, if your terminal only supports `ANSI` or `ANSI256`, try setting custom styles like in the `custom.mojo` example, or update the default profile in `stump/style.mojo` from `TRUE_COLOR` to `ANSI` or `ANSI256`. diff --git a/external/gojo/bufio/__init__.mojo b/external/gojo/bufio/__init__.mojo new file mode 100644 index 0000000..c501992 --- /dev/null +++ b/external/gojo/bufio/__init__.mojo @@ -0,0 +1,2 @@ +from .bufio import Reader, Writer, ReadWriter +from .scan import Scanner, scan_words, scan_bytes, scan_lines diff --git a/external/gojo/bufio/bufio.mojo b/external/gojo/bufio/bufio.mojo new file mode 100644 index 0000000..92fc6b9 --- /dev/null +++ b/external/gojo/bufio/bufio.mojo @@ -0,0 +1,938 @@ +from math import max +from ..io import traits as io +from ..builtins import copy, panic +from ..builtins.bytes import Byte, index_byte +from ..strings import StringBuilder + +alias MIN_READ_BUFFER_SIZE = 16 +alias MAX_CONSECUTIVE_EMPTY_READS = 100 +alias DEFAULT_BUF_SIZE = 4096 + +alias ERR_INVALID_UNREAD_BYTE = "bufio: invalid use of unread_byte" +alias ERR_INVALID_UNREAD_RUNE = "bufio: invalid use of unread_rune" +alias ERR_BUFFER_FULL = "bufio: buffer full" +alias ERR_NEGATIVE_COUNT = "bufio: negative count" +alias ERR_NEGATIVE_READ = "bufio: reader returned negative count from Read" +alias ERR_NEGATIVE_WRITE = "bufio: writer returned negative count from write" + + +# buffered input +struct Reader[R: io.Reader](Sized, io.Reader, io.ByteReader, io.ByteScanner, io.WriterTo): + """Implements buffering for an io.Reader object.""" + + var buf: List[Byte] + var reader: R # reader provided by the client + var read_pos: Int + var write_pos: Int # buf read and write positions + var last_byte: Int # last byte read for unread_byte; -1 means invalid + var last_rune_size: Int # size of last rune read for unread_rune; -1 means invalid + var err: Error + + fn __init__( + inout self, + owned reader: R, + buf: List[Byte] = List[Byte](capacity=DEFAULT_BUF_SIZE), + read_pos: Int = 0, + write_pos: Int = 0, + last_byte: Int = -1, + last_rune_size: Int = -1, + ): + self.buf = buf + self.reader = reader^ + self.read_pos = read_pos + self.write_pos = write_pos + self.last_byte = last_byte + self.last_rune_size = last_rune_size + self.err = Error() + + fn __moveinit__(inout self, owned existing: Self): + self.buf = existing.buf^ + self.reader = existing.reader^ + self.read_pos = existing.read_pos + self.write_pos = existing.write_pos + self.last_byte = existing.last_byte + self.last_rune_size = existing.last_rune_size + self.err = existing.err^ + + # size returns the size of the underlying buffer in bytes. + fn __len__(self) -> Int: + return len(self.buf) + + # reset discards any buffered data, resets all state, and switches + # the buffered reader to read from r. + # Calling reset on the zero value of [Reader] initializes the internal buffer + # to the default size. + # Calling self.reset(b) (that is, resetting a [Reader] to itself) does nothing. + # fn reset[R: io.Reader](self, reader: R): + # # If a Reader r is passed to NewReader, NewReader will return r. + # # Different layers of code may do that, and then later pass r + # # to reset. Avoid infinite recursion in that case. + # if self == reader: + # return + + # # if self.buf == nil: + # # self.buf = make(List[Byte], DEFAULT_BUF_SIZE) + + # self.reset(self.buf, r) + + fn reset(inout self, buf: List[Byte], owned reader: R): + self = Reader[R]( + buf=buf, + reader=reader^, + last_byte=-1, + last_rune_size=-1, + ) + + fn fill(inout self): + """Reads a new chunk into the buffer.""" + # Slide existing data to beginning. + if self.read_pos > 0: + var current_capacity = self.buf.capacity + self.buf = self.buf[self.read_pos : self.write_pos] + self.buf.reserve(current_capacity) + self.write_pos -= self.read_pos + self.read_pos = 0 + + # Compares to the length of the entire List[Byte] object, including 0 initialized positions. + # IE. var b = List[Byte](capacity=4096), then trying to write at b[4096] and onwards will fail. + if self.write_pos >= self.buf.capacity: + panic("bufio.Reader: tried to fill full buffer") + + # Read new data: try a limited number of times. + var i: Int = MAX_CONSECUTIVE_EMPTY_READS + while i > 0: + # TODO: Using temp until slicing can return a Reference + var temp = List[Byte](capacity=DEFAULT_BUF_SIZE) + var bytes_read: Int + var err: Error + bytes_read, err = self.reader.read(temp) + if bytes_read < 0: + panic(ERR_NEGATIVE_READ) + + bytes_read = copy(self.buf, temp, self.write_pos) + self.write_pos += bytes_read + + if err: + self.err = err + return + + if bytes_read > 0: + return + + i -= 1 + + self.err = Error(io.ERR_NO_PROGRESS) + + fn read_error(inout self) -> Error: + if not self.err: + return Error() + + var err = self.err + self.err = Error() + return err + + fn peek(inout self, number_of_bytes: Int) -> (List[Byte], Error): + """Returns the next n bytes without advancing the reader. The bytes stop + being valid at the next read call. If Peek returns fewer than n bytes, it + also returns an error explaining why the read is short. The error is + [ERR_BUFFER_FULL] if number_of_bytes is larger than b's buffer size. + + Calling Peek prevents a [Reader.unread_byte] or [Reader.unread_rune] call from succeeding + until the next read operation. + + Args: + number_of_bytes: The number of bytes to peek. + """ + if number_of_bytes < 0: + return List[Byte](), Error(ERR_NEGATIVE_COUNT) + + self.last_byte = -1 + self.last_rune_size = -1 + + while self.write_pos - self.read_pos < number_of_bytes and self.write_pos - self.read_pos < self.buf.capacity: + self.fill() # self.write_pos-self.read_pos < self.buf.capacity => buffer is not full + + if number_of_bytes > self.buf.capacity: + return self.buf[self.read_pos : self.write_pos], Error(ERR_BUFFER_FULL) + + # 0 <= n <= self.buf.capacity + var err = Error() + var available_space = self.write_pos - self.read_pos + if available_space < number_of_bytes: + # not enough data in buffer + err = self.read_error() + if not err: + err = Error(ERR_BUFFER_FULL) + + return self.buf[self.read_pos : self.read_pos + number_of_bytes], err + + fn discard(inout self, number_of_bytes: Int) -> (Int, Error): + """Discard skips the next n bytes, returning the number of bytes discarded. + + If Discard skips fewer than n bytes, it also returns an error. + If 0 <= number_of_bytes <= self.buffered(), Discard is guaranteed to succeed without + reading from the underlying io.Reader. + """ + if number_of_bytes < 0: + return 0, Error(ERR_NEGATIVE_COUNT) + + if number_of_bytes == 0: + return 0, Error() + + self.last_byte = -1 + self.last_rune_size = -1 + + var remain = number_of_bytes + while True: + var skip = self.buffered() + if skip == 0: + self.fill() + skip = self.buffered() + + if skip > remain: + skip = remain + + self.read_pos += skip + remain -= skip + if remain == 0: + return number_of_bytes, Error() + + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Reads data into dest. + It returns the number of bytes read into dest. + The bytes are taken from at most one Read on the underlying [Reader], + hence n may be less than len(src). + To read exactly len(src) bytes, use io.ReadFull(b, src). + If the underlying [Reader] can return a non-zero count with io.EOF, + then this Read method can do so as well; see the [io.Reader] docs.""" + var space_available = dest.capacity - len(dest) + if space_available == 0: + if self.buffered() > 0: + return 0, Error() + return 0, self.read_error() + + var bytes_read: Int = 0 + if self.read_pos == self.write_pos: + if space_available >= len(self.buf): + # Large read, empty buffer. + # Read directly into dest to avoid copy. + var bytes_read: Int + var err: Error + bytes_read, err = self.reader.read(dest) + + self.err = err + if bytes_read < 0: + panic(ERR_NEGATIVE_READ) + + if bytes_read > 0: + self.last_byte = int(dest[bytes_read - 1]) + self.last_rune_size = -1 + + return bytes_read, self.read_error() + + # One read. + # Do not use self.fill, which will loop. + self.read_pos = 0 + self.write_pos = 0 + var bytes_read: Int + var err: Error + bytes_read, err = self.reader.read(self.buf) + + if bytes_read < 0: + panic(ERR_NEGATIVE_READ) + + if bytes_read == 0: + return 0, self.read_error() + + self.write_pos += bytes_read + + # copy as much as we can + # Note: if the slice panics here, it is probably because + # the underlying reader returned a bad count. See issue 49795. + bytes_read = copy(dest, self.buf[self.read_pos : self.write_pos]) + self.read_pos += bytes_read + self.last_byte = int(self.buf[self.read_pos - 1]) + self.last_rune_size = -1 + return bytes_read, Error() + + fn read_byte(inout self) -> (Byte, Error): + """Reads and returns a single byte from the internal buffer. If no byte is available, returns an error.""" + self.last_rune_size = -1 + while self.read_pos == self.write_pos: + if self.err: + return Int8(0), self.read_error() + self.fill() # buffer is empty + + var c = self.buf[self.read_pos] + self.read_pos += 1 + self.last_byte = int(c) + return c, Error() + + fn unread_byte(inout self) -> Error: + """Unreads the last byte. Only the most recently read byte can be unread. + + unread_byte returns an error if the most recent method called on the + [Reader] was not a read operation. Notably, [Reader.peek], [Reader.discard], and [Reader.write_to] are not + considered read operations. + """ + if self.last_byte < 0 or self.read_pos == 0 and self.write_pos > 0: + return Error(ERR_INVALID_UNREAD_BYTE) + + # self.read_pos > 0 or self.write_pos == 0 + if self.read_pos > 0: + self.read_pos -= 1 + else: + # self.read_pos == 0 and self.write_pos == 0 + self.write_pos = 1 + + self.buf[self.read_pos] = self.last_byte + self.last_byte = -1 + self.last_rune_size = -1 + return Error() + + # # read_rune reads a single UTF-8 encoded Unicode character and returns the + # # rune and its size in bytes. If the encoded rune is invalid, it consumes one byte + # # and returns unicode.ReplacementChar (U+FFFD) with a size of 1. + # fn read_rune(inout self) (r rune, size int, err error): + # for self.read_pos+utf8.UTFMax > self.write_pos and !utf8.FullRune(self.buf[self.read_pos:self.write_pos]) and self.err == nil and self.write_pos-self.read_pos < self.buf.capacity: + # self.fill() # self.write_pos-self.read_pos < len(buf) => buffer is not full + + # self.last_rune_size = -1 + # if self.read_pos == self.write_pos: + # return 0, 0, self.read_poseadErr() + + # r, size = rune(self.buf[self.read_pos]), 1 + # if r >= utf8.RuneSelf: + # r, size = utf8.DecodeRune(self.buf[self.read_pos:self.write_pos]) + + # self.read_pos += size + # self.last_byte = int(self.buf[self.read_pos-1]) + # self.last_rune_size = size + # return r, size, nil + + # # unread_rune unreads the last rune. If the most recent method called on + # # the [Reader] was not a [Reader.read_rune], [Reader.unread_rune] returns an error. (In this + # # regard it is stricter than [Reader.unread_byte], which will unread the last byte + # # from any read operation.) + # fn unread_rune() error: + # if self.last_rune_size < 0 or self.read_pos < self.last_rune_size: + # return ERR_INVALID_UNREAD_RUNE + + # self.read_pos -= self.last_rune_size + # self.last_byte = -1 + # self.last_rune_size = -1 + # return nil + + fn buffered(self) -> Int: + """Returns the number of bytes that can be read from the current buffer. + + Returns: + The number of bytes that can be read from the current buffer. + """ + return self.write_pos - self.read_pos + + fn read_slice(inout self, delim: Int8) -> (List[Byte], Error): + """Reads until the first occurrence of delim in the input, + returning a slice pointing at the bytes in the buffer. It includes the first occurrence of the delimiter. + The bytes stop being valid at the next read. + If read_slice encounters an error before finding a delimiter, + it returns all the data in the buffer and the error itself (often io.EOF). + read_slice fails with error [ERR_BUFFER_FULL] if the buffer fills without a delim. + Because the data returned from read_slice will be overwritten + by the next I/O operation, most clients should use + [Reader.read_bytes] or read_string instead. + read_slice returns err != nil if and only if line does not end in delim. + + Args: + delim: The delimiter to search for. + + Returns: + The List[Byte] from the internal buffer. + """ + var err = Error() + var s = 0 # search start index + var line: List[Byte] = List[Byte](capacity=DEFAULT_BUF_SIZE) + while True: + # Search buffer. + var i = index_byte(self.buf[self.read_pos + s : self.write_pos], delim) + if i >= 0: + i += s + line = self.buf[self.read_pos : self.read_pos + i + 1] + self.read_pos += i + 1 + break + + # Pending error? + if self.err: + line = self.buf[self.read_pos : self.write_pos] + self.read_pos = self.write_pos + err = self.read_error() + break + + # Buffer full? + if self.buffered() >= self.buf.capacity: + self.read_pos = self.write_pos + line = self.buf + err = Error(ERR_BUFFER_FULL) + break + + s = self.write_pos - self.read_pos # do not rescan area we scanned before + self.fill() # buffer is not full + + # Handle last byte, if any. + var i = len(line) - 1 + if i >= 0: + self.last_byte = int(line[i]) + self.last_rune_size = -1 + + return line, err + + fn read_line(inout self) raises -> (List[Byte], Bool): + """Low-level line-reading primitive. Most callers should use + [Reader.read_bytes]('\n') or [Reader.read_string]('\n') instead or use a [Scanner]. + + read_line tries to return a single line, not including the end-of-line bytes. + If the line was too long for the buffer then isPrefix is set and the + beginning of the line is returned. The rest of the line will be returned + from future calls. isPrefix will be false when returning the last fragment + of the line. The returned buffer is only valid until the next call to + read_line. read_line either returns a non-nil line or it returns an error, + never both. + + The text returned from read_line does not include the line end ("\r\n" or "\n"). + No indication or error is given if the input ends without a final line end. + Calling [Reader.unread_byte] after read_line will always unread the last byte read + (possibly a character belonging to the line end) even if that byte is not + part of the line returned by read_line. + """ + var line: List[Byte] + var err: Error + line, err = self.read_slice(ord("\n")) + + if err and str(err) == ERR_BUFFER_FULL: + # Handle the case where "\r\n" straddles the buffer. + if len(line) > 0 and line[len(line) - 1] == ord("\r"): + # Put the '\r' back on buf and drop it from line. + # Let the next call to read_line check for "\r\n". + if self.read_pos == 0: + # should be unreachable + raise Error("bufio: tried to rewind past start of buffer") + + self.read_pos -= 1 + line = line[: len(line) - 1] + return line, True + + if len(line) == 0: + return line, False + + if line[len(line) - 1] == ord("\n"): + var drop = 1 + if len(line) > 1 and line[len(line) - 2] == ord("\r"): + drop = 2 + + line = line[: len(line) - drop] + + return line, False + + fn collect_fragments(inout self, delim: Int8) -> (List[List[Byte]], List[Byte], Int, Error): + """Reads until the first occurrence of delim in the input. It + returns (slice of full buffers, remaining bytes before delim, total number + of bytes in the combined first two elements, error). + + Args: + delim: The delimiter to search for. + """ + # Use read_slice to look for delim, accumulating full buffers. + var err = Error() + var full_buffers = List[List[Byte]]() + var total_len = 0 + var frag = List[Byte](capacity=4096) + while True: + frag, err = self.read_slice(delim) + if not err: + break + + var read_slice_error = err + if str(read_slice_error) != ERR_BUFFER_FULL: + err = read_slice_error + break + + # Make a copy of the buffer. + var buf = List[Byte](frag) + full_buffers.append(buf) + total_len += len(buf) + + total_len += len(frag) + return full_buffers, frag, total_len, err + + fn read_bytes(inout self, delim: Int8) -> (List[Byte], Error): + """Reads until the first occurrence of delim in the input, + returning a slice containing the data up to and including the delimiter. + If read_bytes encounters an error before finding a delimiter, + it returns the data read before the error and the error itself (often io.EOF). + read_bytes returns err != nil if and only if the returned data does not end in + delim. + For simple uses, a Scanner may be more convenient. + + Args: + delim: The delimiter to search for. + + Returns: + The List[Byte] from the internal buffer. + """ + var full: List[List[Byte]] + var frag: List[Byte] + var n: Int + var err: Error + full, frag, n, err = self.collect_fragments(delim) + + # Allocate new buffer to hold the full pieces and the fragment. + var buf = List[Byte](capacity=n) + n = 0 + + # copy full pieces and fragment in. + for i in range(len(full)): + var buffer = full[i] + n += copy(buf, buffer, n) + + _ = copy(buf, frag, n) + + return buf, err + + fn read_string(inout self, delim: Int8) -> (String, Error): + """Reads until the first occurrence of delim in the input, + returning a string containing the data up to and including the delimiter. + If read_string encounters an error before finding a delimiter, + it returns the data read before the error and the error itself (often io.EOF). + read_string returns err != nil if and only if the returned data does not end in + delim. + For simple uses, a Scanner may be more convenient. + + Args: + delim: The delimiter to search for. + + Returns: + The String from the internal buffer. + """ + var full: List[List[Byte]] + var frag: List[Byte] + var n: Int + var err: Error + full, frag, n, err = self.collect_fragments(delim) + + # Allocate new buffer to hold the full pieces and the fragment. + var buf = StringBuilder(size=n) + + # copy full pieces and fragment in. + for i in range(len(full)): + var buffer = full[i] + _ = buf.write(buffer) + + _ = buf.write(frag) + return str(buf), err + + fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): + """Writes the internal buffer to the writer. This may make multiple calls to the [Reader.Read] method of the underlying [Reader]. + If the underlying reader supports the [Reader.WriteTo] method, + this calls the underlying [Reader.WriteTo] without buffering. + write_to implements io.WriterTo. + + Args: + writer: The writer to write to. + + Returns: + The number of bytes written. + """ + self.last_byte = -1 + self.last_rune_size = -1 + + var bytes_written: Int64 + var err: Error + bytes_written, err = self.write_buf(writer) + if err: + return bytes_written, err + + # internal buffer not full, fill before writing to writer + if (self.write_pos - self.read_pos) < self.buf.capacity: + self.fill() + + while self.read_pos < self.write_pos: + # self.read_pos < self.write_pos => buffer is not empty + var bw: Int64 + var err: Error + bw, err = self.write_buf(writer) + bytes_written += bw + + self.fill() # buffer is empty + + return bytes_written, Error() + + fn write_buf[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): + """Writes the [Reader]'s buffer to the writer. + + Args: + writer: The writer to write to. + + Returns: + The number of bytes written. + """ + # Nothing to write + if self.read_pos == self.write_pos: + return Int64(0), Error() + + # Write the buffer to the writer, if we hit EOF it's fine. That's not a failure condition. + var bytes_written: Int + var err: Error + bytes_written, err = writer.write(self.buf[self.read_pos : self.write_pos]) + if err: + return Int64(bytes_written), err + + if bytes_written < 0: + panic(ERR_NEGATIVE_WRITE) + + self.read_pos += bytes_written + return Int64(bytes_written), Error() + + +# fn new_reader_size[R: io.Reader](owned reader: R, size: Int) -> Reader[R]: +# """Returns a new [Reader] whose buffer has at least the specified +# size. If the argument io.Reader is already a [Reader] with large enough +# size, it returns the underlying [Reader]. + +# Args: +# reader: The reader to read from. +# size: The size of the buffer. + +# Returns: +# The new [Reader]. +# """ +# # # Is it already a Reader? +# # b, ok := rd.(*Reader) +# # if ok and self.buf.capacity >= size: +# # return b + +# var r = Reader(reader ^) +# r.reset(List[Byte](capacity=max(size, MIN_READ_BUFFER_SIZE)), reader ^) +# return r + + +# fn new_reader[R: io.Reader](reader: R) -> Reader[R]: +# """Returns a new [Reader] whose buffer has the default size. + +# Args: +# reader: The reader to read from. + +# Returns: +# The new [Reader]. +# """ +# return new_reader_size(reader, DEFAULT_BUF_SIZE) + + +# buffered output +# TODO: Reader and Writer maybe should not take ownership of the underlying reader/writer? Seems okay for now. +struct Writer[W: io.Writer](Sized, io.Writer, io.ByteWriter, io.StringWriter, io.ReaderFrom): + """Implements buffering for an [io.Writer] object. + # If an error occurs writing to a [Writer], no more data will be + # accepted and all subsequent writes, and [Writer.flush], will return the error. + # After all data has been written, the client should call the + # [Writer.flush] method to guarantee all data has been forwarded to + # the underlying [io.Writer].""" + + var buf: List[Byte] + var bytes_written: Int + var writer: W + var err: Error + + fn __init__( + inout self, + owned writer: W, + buf: List[Byte] = List[Byte](capacity=DEFAULT_BUF_SIZE), + bytes_written: Int = 0, + ): + self.buf = buf + self.bytes_written = bytes_written + self.writer = writer^ + self.err = Error() + + fn __moveinit__(inout self, owned existing: Self): + self.buf = existing.buf^ + self.bytes_written = existing.bytes_written + self.writer = existing.writer^ + self.err = existing.err^ + + fn __len__(self) -> Int: + """Returns the size of the underlying buffer in bytes.""" + return len(self.buf) + + fn reset(inout self, owned writer: W): + """Discards any unflushed buffered data, clears any error, and + resets b to write its output to w. + Calling reset on the zero value of [Writer] initializes the internal buffer + to the default size. + Calling w.reset(w) (that is, resetting a [Writer] to itself) does nothing. + + Args: + writer: The writer to write to. + """ + # # If a Writer w is passed to new_writer, new_writer will return w. + # # Different layers of code may do that, and then later pass w + # # to reset. Avoid infinite recursion in that case. + # if self == writer: + # return + + # if self.buf == nil: + # self.buf = make(List[Byte], DEFAULT_BUF_SIZE) + + self.err = Error() + self.bytes_written = 0 + self.writer = writer^ + + fn flush(inout self) -> Error: + """Writes any buffered data to the underlying [io.Writer].""" + # Prior to attempting to flush, check if there's a pre-existing error or if there's nothing to flush. + var err = Error() + if self.err: + return self.err + if self.bytes_written == 0: + return err + + var bytes_written: Int = 0 + bytes_written, err = self.writer.write(self.buf[0 : self.bytes_written]) + + # If the write was short, set a short write error and try to shift up the remaining bytes. + if bytes_written < self.bytes_written and not err: + err = Error(io.ERR_SHORT_WRITE) + + if err: + if bytes_written > 0 and bytes_written < self.bytes_written: + _ = copy(self.buf, self.buf[bytes_written : self.bytes_written]) + + self.bytes_written -= bytes_written + self.err = err + return err + + # Reset the buffer + self.buf = List[Byte](capacity=self.buf.capacity) + self.bytes_written = 0 + return err + + fn available(self) -> Int: + """Returns how many bytes are unused in the buffer.""" + return self.buf.capacity - len(self.buf) + + fn available_buffer(self) raises -> List[Byte]: + """Returns an empty buffer with self.available() capacity. + This buffer is intended to be appended to and + passed to an immediately succeeding [Writer.write] call. + The buffer is only valid until the next write operation on self. + + Returns: + An empty buffer with self.available() capacity. + """ + return self.buf[self.bytes_written :][:0] + + fn buffered(self) -> Int: + """Returns the number of bytes that have been written into the current buffer. + + Returns: + The number of bytes that have been written into the current buffer. + """ + return self.bytes_written + + fn write(inout self, src: List[Byte]) -> (Int, Error): + """Writes the contents of src into the buffer. + It returns the number of bytes written. + If nn < len(src), it also returns an error explaining + why the write is short. + + Args: + src: The bytes to write. + + Returns: + The number of bytes written. + """ + var total_bytes_written: Int = 0 + var src_copy = src + var err = Error() + while len(src_copy) > self.available() and not self.err: + var bytes_written: Int = 0 + if self.buffered() == 0: + # Large write, empty buffer. + # write directly from p to avoid copy. + bytes_written, err = self.writer.write(src_copy) + self.err = err + else: + bytes_written = copy(self.buf, src_copy, self.bytes_written) + self.bytes_written += bytes_written + _ = self.flush() + + total_bytes_written += bytes_written + src_copy = src_copy[bytes_written : len(src_copy)] + + if self.err: + return total_bytes_written, self.err + + var n = copy(self.buf, src_copy, self.bytes_written) + self.bytes_written += n + total_bytes_written += n + return total_bytes_written, err + + fn write_byte(inout self, src: Int8) -> (Int, Error): + """Writes a single byte to the internal buffer. + + Args: + src: The byte to write. + """ + if self.err: + return 0, self.err + # If buffer is full, flush to the underlying writer. + var err = self.flush() + if self.available() <= 0 and err: + return 0, self.err + + self.buf.append(src) + self.bytes_written += 1 + + return 1, Error() + + # # WriteRune writes a single Unicode code point, returning + # # the number of bytes written and any error. + # fn WriteRune(r rune) (size int, err error): + # # Compare as uint32 to correctly handle negative runes. + # if uint32(r) < utf8.RuneSelf: + # err = self.write_posriteByte(byte(r)) + # if err != nil: + # return 0, err + + # return 1, nil + + # if self.err != nil: + # return 0, self.err + + # n := self.available() + # if n < utf8.UTFMax: + # if self.flush(); self.err != nil: + # return 0, self.err + + # n = self.available() + # if n < utf8.UTFMax: + # # Can only happen if buffer is silly small. + # return self.write_posriteString(string(r)) + + # size = utf8.EncodeRune(self.buf[self.bytes_written:], r) + # self.bytes_written += size + # return size, nil + + fn write_string(inout self, src: String) -> (Int, Error): + """Writes a string to the internal buffer. + It returns the number of bytes written. + If the count is less than len(s), it also returns an error explaining + why the write is short. + + Args: + src: The string to write. + + Returns: + The number of bytes written. + """ + return self.write(src.as_bytes()) + + fn read_from[R: io.Reader](inout self, inout reader: R) -> (Int64, Error): + """Implements [io.ReaderFrom]. If the underlying writer + supports the read_from method, this calls the underlying read_from. + If there is buffered data and an underlying read_from, this fills + the buffer and writes it before calling read_from. + + Args: + reader: The reader to read from. + + Returns: + The number of bytes read. + """ + if self.err: + return Int64(0), self.err + + var bytes_read: Int = 0 + var total_bytes_written: Int64 = 0 + var err = Error() + while True: + if self.available() == 0: + var err = self.flush() + if err: + return total_bytes_written, err + + var nr = 0 + while nr < MAX_CONSECUTIVE_EMPTY_READS: + # TODO: should really be using a slice that returns refs and not a copy. + # Read into remaining unused space in the buffer. We need to reserve capacity for the slice otherwise read will never hit EOF. + var sl = self.buf[self.bytes_written : len(self.buf)] + sl.reserve(self.buf.capacity) + bytes_read, err = reader.read(sl) + if bytes_read > 0: + bytes_read = copy(self.buf, sl, self.bytes_written) + + if bytes_read != 0 or err: + break + nr += 1 + + if nr == MAX_CONSECUTIVE_EMPTY_READS: + return Int64(bytes_read), Error(io.ERR_NO_PROGRESS) + + self.bytes_written += bytes_read + total_bytes_written += Int64(bytes_read) + if err: + break + + if err and str(err) == io.EOF: + # If we filled the buffer exactly, flush preemptively. + if self.available() == 0: + err = self.flush() + else: + err = Error() + + return total_bytes_written, Error() + + +fn new_writer_size[W: io.Writer](owned writer: W, size: Int) -> Writer[W]: + """Returns a new [Writer] whose buffer has at least the specified + size. If the argument io.Writer is already a [Writer] with large enough + size, it returns the underlying [Writer].""" + # Is it already a Writer? + # b, ok := w.(*Writer) + # if ok and self.buf.capacity >= size: + # return b + + var buf_size = size + if buf_size <= 0: + buf_size = DEFAULT_BUF_SIZE + + return Writer[W]( + buf=List[Byte](capacity=size), + writer=writer^, + bytes_written=0, + ) + + +fn new_writer[W: io.Writer](owned writer: W) -> Writer[W]: + """Returns a new [Writer] whose buffer has the default size. + # If the argument io.Writer is already a [Writer] with large enough buffer size, + # it returns the underlying [Writer].""" + return new_writer_size[W](writer^, DEFAULT_BUF_SIZE) + + +# buffered input and output +struct ReadWriter[R: io.Reader, W: io.Writer](): + """ReadWriter stores pointers to a [Reader] and a [Writer]. + It implements [io.ReadWriter].""" + + var reader: R + var writer: W + + fn __init__(inout self, owned reader: R, owned writer: W): + self.reader = reader^ + self.writer = writer^ + + +# new_read_writer +fn new_read_writer[R: io.Reader, W: io.Writer](owned reader: R, owned writer: W) -> ReadWriter[R, W]: + """Allocates a new [ReadWriter] that dispatches to r and w.""" + return ReadWriter[R, W](reader^, writer^) diff --git a/external/gojo/bufio/scan.mojo b/external/gojo/bufio/scan.mojo new file mode 100644 index 0000000..28489fc --- /dev/null +++ b/external/gojo/bufio/scan.mojo @@ -0,0 +1,458 @@ +import math +from collections import Optional +import ..io +from ..builtins import copy, panic, Error +from ..builtins.bytes import Byte, index_byte +from .bufio import MAX_CONSECUTIVE_EMPTY_READS + + +alias MAX_INT: Int = 2147483647 + + +struct Scanner[R: io.Reader](): + """Scanner provides a convenient Interface for reading data such as + a file of newline-delimited lines of text. Successive calls to + the [Scanner.Scan] method will step through the 'tokens' of a file, skipping + the bytes between the tokens. The specification of a token is + defined by a split function of type [SplitFunction]; the default split + function breaks the input Into lines with line termination stripped. [Scanner.split] + fntions are defined in this package for scanning a file Into + lines, bytes, UTF-8-encoded runes, and space-delimited words. The + client may instead provide a custom split function. + + Scanning stops unrecoverably at EOF, the first I/O error, or a token too + large to fit in the [Scanner.buffer]. When a scan stops, the reader may have + advanced arbitrarily far past the last token. Programs that need more + control over error handling or large tokens, or must run sequential scans + on a reader, should use [bufio.Reader] instead.""" + + var reader: R # The reader provided by the client. + var split: SplitFunction # The function to split the tokens. + var max_token_size: Int # Maximum size of a token; modified by tests. + var token: List[Byte] # Last token returned by split. + var buf: List[Byte] # buffer used as argument to split. + var start: Int # First non-processed byte in buf. + var end: Int # End of data in buf. + var empties: Int # Count of successive empty tokens. + var scan_called: Bool # Scan has been called; buffer is in use. + var done: Bool # Scan has finished. + var err: Error + + fn __init__( + inout self, + owned reader: R, + split: SplitFunction = scan_lines, + max_token_size: Int = MAX_SCAN_TOKEN_SIZE, + token: List[Byte] = List[Byte](capacity=io.BUFFER_SIZE), + buf: List[Byte] = List[Byte](capacity=io.BUFFER_SIZE), + start: Int = 0, + end: Int = 0, + empties: Int = 0, + scan_called: Bool = False, + done: Bool = False, + ): + self.reader = reader^ + self.split = split + self.max_token_size = max_token_size + self.token = token + self.buf = buf + self.start = start + self.end = end + self.empties = empties + self.scan_called = scan_called + self.done = done + self.err = Error() + + fn current_token_as_bytes(self) -> List[Byte]: + """Returns the most recent token generated by a call to [Scanner.Scan]. + The underlying array may point to data that will be overwritten + by a subsequent call to Scan. It does no allocation. + """ + return self.token + + fn current_token(self) -> String: + """Returns the most recent token generated by a call to [Scanner.Scan] + as a newly allocated string holding its bytes.""" + return String(self.token) + + fn scan(inout self) raises -> Bool: + """Advances the [Scanner] to the next token, which will then be + available through the [Scanner.current_token_as_bytes] or [Scanner.current_token] method. + It returns False when there are no more tokens, either by reaching the end of the input or an error. + After Scan returns False, the [Scanner.Err] method will return any error that + occurred during scanning, except if it was [io.EOF], [Scanner.Err]. + Scan raises an Error if the split function returns too many empty + tokens without advancing the input. This is a common error mode for + scanners. + """ + if self.done: + return False + + self.scan_called = True + # Loop until we have a token. + while True: + # See if we can get a token with what we already have. + # If we've run out of data but have an error, give the split function + # a chance to recover any remaining, possibly empty token. + if (self.end > self.start) or self.err: + var advance: Int + var token = List[Byte](capacity=io.BUFFER_SIZE) + var err = Error() + var at_eof = False + if self.err: + at_eof = True + advance, token, err = self.split(self.buf[self.start : self.end], at_eof) + if err: + if str(err) == ERR_FINAL_TOKEN: + self.token = token + self.done = True + # When token is not nil, it means the scanning stops + # with a trailing token, and thus the return value + # should be True to indicate the existence of the token. + return len(token) != 0 + + self.set_err(err) + return False + + if not self.advance(advance): + return False + + self.token = token + if len(token) != 0: + if not self.err or advance > 0: + self.empties = 0 + else: + # Returning tokens not advancing input at EOF. + self.empties += 1 + if self.empties > MAX_CONSECUTIVE_EMPTY_READS: + panic("bufio.Scan: too many empty tokens without progressing") + + return True + + # We cannot generate a token with what we are holding. + # If we've already hit EOF or an I/O error, we are done. + if self.err: + # Shut it down. + self.start = 0 + self.end = 0 + return False + + # Must read more data. + # First, shift data to beginning of buffer if there's lots of empty space + # or space is needed. + if self.start > 0 and (self.end == len(self.buf) or self.start > int(len(self.buf) / 2)): + _ = copy(self.buf, self.buf[self.start : self.end]) + self.end -= self.start + self.start = 0 + + # Is the buffer full? If so, resize. + if self.end == len(self.buf): + # Guarantee no overflow in the multiplication below. + if len(self.buf) >= self.max_token_size or len(self.buf) > int(MAX_INT / 2): + self.set_err(Error(ERR_TOO_LONG)) + return False + + var new_size = len(self.buf) * 2 + if new_size == 0: + new_size = START_BUF_SIZE + + # Make a new List[Byte] buffer and copy the elements in + new_size = math.min(new_size, self.max_token_size) + var new_buf = List[Byte](capacity=new_size) + _ = copy(new_buf, self.buf[self.start : self.end]) + self.buf = new_buf + self.end -= self.start + self.start = 0 + + # Finally we can read some input. Make sure we don't get stuck with + # a misbehaving Reader. Officially we don't need to do this, but let's + # be extra careful: Scanner is for safe, simple jobs. + var loop = 0 + while True: + var bytes_read: Int + var sl = self.buf[self.end : len(self.buf)] + var err: Error + + # Catch any reader errors and set the internal error field to that err instead of bubbling it up. + bytes_read, err = self.reader.read(sl) + _ = copy(self.buf, sl, self.end) + if bytes_read < 0 or len(self.buf) - self.end < bytes_read: + self.set_err(Error(ERR_BAD_READ_COUNT)) + break + + self.end += bytes_read + if err: + self.set_err(err) + break + + if bytes_read > 0: + self.empties = 0 + break + + loop += 1 + if loop > MAX_CONSECUTIVE_EMPTY_READS: + self.set_err(Error(io.ERR_NO_PROGRESS)) + break + + fn set_err(inout self, err: Error): + """Set the internal error field to the provided error. + + Args: + err: The error to set. + """ + if self.err: + var value = String(self.err) + if value == "" or value == io.EOF: + self.err = err + else: + self.err = err + + fn advance(inout self, n: Int) -> Bool: + """Consumes n bytes of the buffer. It reports whether the advance was legal. + + Args: + n: The number of bytes to advance the buffer by. + + Returns: + True if the advance was legal, False otherwise. + """ + if n < 0: + self.set_err(Error(ERR_NEGATIVE_ADVANCE)) + return False + + if n > self.end - self.start: + self.set_err(Error(ERR_ADVANCE_TOO_FAR)) + return False + + self.start += n + return True + + fn buffer(inout self, buf: List[Byte], max: Int) raises: + """Sets the initial buffer to use when scanning + and the maximum size of buffer that may be allocated during scanning. + The maximum token size must be less than the larger of max and cap(buf). + If max <= cap(buf), [Scanner.Scan] will use this buffer only and do no allocation. + + By default, [Scanner.Scan] uses an Internal buffer and sets the + maximum token size to [MAX_SCAN_TOKEN_SIZE]. + + buffer raises an Error if it is called after scanning has started. + + Args: + buf: The buffer to use when scanning. + max: The maximum size of buffer that may be allocated during scanning. + + Raises: + Error: If called after scanning has started. + """ + if self.scan_called: + raise Error("buffer called after Scan") + + # self.buf = buf[0:buf.capacity()] + self.max_token_size = max + + # # split sets the split function for the [Scanner]. + # # The default split function is [scan_lines]. + # # + # # split panics if it is called after scanning has started. + # fn split(inout self, split_function: SplitFunction) raises: + # if self.scan_called: + # raise Error("split called after Scan") + + # self.split = split_function + + +# SplitFunction is the signature of the split function used to tokenize the +# input. The arguments are an initial substring of the remaining unprocessed +# data and a flag, at_eof, that reports whether the [Reader] has no more data +# to give. The return values are the number of bytes to advance the input +# and the next token to return to the user, if any, plus an error, if any. +# +# Scanning stops if the function returns an error, in which case some of +# the input may be discarded. If that error is [ERR_FINAL_TOKEN], scanning +# stops with no error. A non-nil token delivered with [ERR_FINAL_TOKEN] +# will be the last token, and a nil token with [ERR_FINAL_TOKEN] +# immediately stops the scanning. +# +# Otherwise, the [Scanner] advances the input. If the token is not nil, +# the [Scanner] returns it to the user. If the token is nil, the +# Scanner reads more data and continues scanning; if there is no more +# data--if at_eof was True--the [Scanner] returns. If the data does not +# yet hold a complete token, for instance if it has no newline while +# scanning lines, a [SplitFunction] can return (0, nil, nil) to signal the +# [Scanner] to read more data Into the slice and try again with a +# longer slice starting at the same poInt in the input. +# +# The function is never called with an empty data slice unless at_eof +# is True. If at_eof is True, however, data may be non-empty and, +# as always, holds unprocessed text. +alias SplitFunction = fn (data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error) + +# # Errors returned by Scanner. +alias ERR_TOO_LONG = Error("bufio.Scanner: token too long") +alias ERR_NEGATIVE_ADVANCE = Error("bufio.Scanner: SplitFunction returns negative advance count") +alias ERR_ADVANCE_TOO_FAR = Error("bufio.Scanner: SplitFunction returns advance count beyond input") +alias ERR_BAD_READ_COUNT = Error("bufio.Scanner: Read returned impossible count") +# ERR_FINAL_TOKEN is a special sentinel error value. It is Intended to be +# returned by a split function to indicate that the scanning should stop +# with no error. If the token being delivered with this error is not nil, +# the token is the last token. +# +# The value is useful to stop processing early or when it is necessary to +# deliver a final empty token (which is different from a nil token). +# One could achieve the same behavior with a custom error value but +# providing one here is tidier. +# See the emptyFinalToken example for a use of this value. +alias ERR_FINAL_TOKEN = Error("final token") + + +# MAX_SCAN_TOKEN_SIZE is the maximum size used to buffer a token +# unless the user provides an explicit buffer with [Scanner.buffer]. +# The actual maximum token size may be smaller as the buffer +# may need to include, for instance, a newline. +alias MAX_SCAN_TOKEN_SIZE = 64 * 1024 +alias START_BUF_SIZE = 4096 # Size of initial allocation for buffer. + + +fn new_scanner[R: io.Reader](owned reader: R) -> Scanner[R]: + """Returns a new [Scanner] to read from r. + The split function defaults to [scan_lines].""" + return Scanner(reader^) + + +###### split functions ###### +fn scan_bytes(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): + """Split function for a [Scanner] that returns each byte as a token.""" + if at_eof and data.capacity == 0: + return 0, List[Byte](), Error() + + return 1, data[0:1], Error() + + +# var errorRune = List[Byte](string(utf8.RuneError)) + +# # ScanRunes is a split function for a [Scanner] that returns each +# # UTF-8-encoded rune as a token. The sequence of runes returned is +# # equivalent to that from a range loop over the input as a string, which +# # means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd". +# # Because of the Scan Interface, this makes it impossible for the client to +# # distinguish correctly encoded replacement runes from encoding errors. +# fn ScanRunes(data List[Byte], at_eof Bool) (advance Int, token List[Byte], err error): +# if at_eof and data.capacity == 0: +# return 0, nil, nil + + +# # Fast path 1: ASCII. +# if data[0] < utf8.RuneSelf: +# return 1, data[0:1], nil + + +# # Fast path 2: Correct UTF-8 decode without error. +# _, width := utf8.DecodeRune(data) +# if width > 1: +# # It's a valid encoding. Width cannot be one for a correctly encoded +# # non-ASCII rune. +# return width, data[0:width], nil + + +# # We know it's an error: we have width==1 and implicitly r==utf8.RuneError. +# # Is the error because there wasn't a full rune to be decoded? +# # FullRune distinguishes correctly between erroneous and incomplete encodings. +# if !at_eof and !utf8.FullRune(data): +# # Incomplete; get more bytes. +# return 0, nil, nil + + +# # We have a real UTF-8 encoding error. Return a properly encoded error rune +# # but advance only one byte. This matches the behavior of a range loop over +# # an incorrectly encoded string. +# return 1, errorRune, nil + + +fn drop_carriage_return(data: List[Byte]) -> List[Byte]: + """Drops a terminal \r from the data. + + Args: + data: The data to strip. + + Returns: + The stripped data. + """ + # In the case of a \r ending without a \n, indexing on -1 doesn't work as it finds a null terminator instead of \r. + if data.capacity > 0 and data[data.capacity - 1] == ord("\r"): + return data[0 : data.capacity - 1] + + return data + + +# TODO: Doing modification of token and err in these split functions, so we don't have to return any memory only types as part of the return tuple. +fn scan_lines(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): + """Split function for a [Scanner] that returns each line of + text, stripped of any trailing end-of-line marker. The returned line may + be empty. The end-of-line marker is one optional carriage return followed + by one mandatory newline. The last non-empty line of input will be returned even if it has no + newline. + + Args: + data: The data to split. + at_eof: Whether the data is at the end of the file. + Returns: + The number of bytes to advance the input. + """ + if at_eof and data.capacity == 0: + return 0, List[Byte](), Error() + + var i = index_byte(data, ord("\n")) + if i >= 0: + # We have a full newline-terminated line. + return i + 1, drop_carriage_return(data[0:i]), Error() + + # If we're at EOF, we have a final, non-terminated line. Return it. + # if at_eof: + return data.capacity, drop_carriage_return(data), Error() + + # Request more data. + # return 0 + + +fn is_space(r: Int8) -> Bool: + alias ALL_WHITESPACES: String = " \t\n\r\x0b\f" + if chr(int(r)) in ALL_WHITESPACES: + return True + return False + + +# TODO: Handle runes and utf8 decoding. For now, just assuming single byte length. +fn scan_words(data: List[Byte], at_eof: Bool) -> (Int, List[Byte], Error): + """Split function for a [Scanner] that returns each + space-separated word of text, with surrounding spaces deleted. It will + never return an empty string. The definition of space is set by + unicode.IsSpace. + """ + # Skip leading spaces. + var start = 0 + var width = 0 + while start < data.capacity: + width = len(data[0]) + if not is_space(data[0]): + break + + start += width + + # Scan until space, marking end of word. + var i = 0 + width = 0 + start = 0 + while i < data.capacity: + width = len(data[i]) + if is_space(data[i]): + return i + width, data[start:i], Error() + + i += width + + # If we're at EOF, we have a final, non-empty, non-terminated word. Return it. + if at_eof and data.capacity > start: + return data.capacity, data[start:], Error() + + # Request more data. + return start, List[Byte](), Error() diff --git a/external/gojo/bytes/__init__.mojo b/external/gojo/bytes/__init__.mojo new file mode 100644 index 0000000..15170f2 --- /dev/null +++ b/external/gojo/bytes/__init__.mojo @@ -0,0 +1,2 @@ +from .buffer import Buffer, new_buffer +from .reader import Reader, new_reader diff --git a/external/gojo/bytes/buffer.mojo b/external/gojo/bytes/buffer.mojo new file mode 100644 index 0000000..13f2df9 --- /dev/null +++ b/external/gojo/bytes/buffer.mojo @@ -0,0 +1,648 @@ +from ..io import ( + Reader, + Writer, + ReadWriter, + ByteReader, + ByteWriter, + WriterTo, + StringWriter, + ReaderFrom, + BUFFER_SIZE, +) +from ..builtins import cap, copy, Byte, panic, index_byte + + +alias Rune = Int32 + +# SMALL_BUFFER_SIZE is an initial allocation minimal capacity. +alias SMALL_BUFFER_SIZE: Int = 64 + +# The ReadOp constants describe the last action performed on +# the buffer, so that unread_rune and unread_byte can check for +# invalid usage. op_read_runeX constants are chosen such that +# converted to Int they correspond to the rune size that was read. +alias ReadOp = Int8 + +# Don't use iota for these, as the values need to correspond with the +# names and comments, which is easier to see when being explicit. +alias OP_READ: ReadOp = -1 # Any other read operation. +alias OP_INVALID: ReadOp = 0 # Non-read operation. +alias OP_READ_RUNE1: ReadOp = 1 # read rune of size 1. +alias OP_READ_RUNE2: ReadOp = 2 # read rune of size 2. +alias OP_READ_RUNE3: ReadOp = 3 # read rune of size 3. +alias OP_READ_RUNE4: ReadOp = 4 # read rune of size 4. + +alias MAX_INT: Int = 2147483647 +# MIN_READ is the minimum slice size passed to a read call by +# [Buffer.read_from]. As long as the [Buffer] has at least MIN_READ bytes beyond +# what is required to hold the contents of r, read_from will not grow the +# underlying buffer. +alias MIN_READ: Int = 512 + +# ERR_TOO_LARGE is passed to panic if memory cannot be allocated to store data in a buffer. +alias ERR_TOO_LARGE = "buffer.Buffer: too large" +alias ERR_NEGATIVE_READ = "buffer.Buffer: reader returned negative count from read" +alias ERR_SHORT_WRITE = "short write" + + +@value +struct Buffer( + Copyable, + Stringable, + Sized, + ReadWriter, + StringWriter, + ByteReader, + ByteWriter, + WriterTo, + ReaderFrom, +): + """A Buffer is a variable-sized buffer of bytes with [Buffer.read] and [Buffer.write] methods. + The zero value for Buffer is an empty buffer ready to use. + """ + + var buf: List[Byte] # contents are the bytes buf[off : len(buf)] + var off: Int # read at &buf[off], write at &buf[len(buf)] + var last_read: ReadOp # last read operation, so that unread* can work correctly. + + fn __init__(inout self, owned buf: List[Byte]): + self.buf = buf + self.off = 0 + self.last_read = OP_INVALID + + fn bytes(self) -> List[Byte]: + """Returns a slice of length self.buf.capacity holding the unread portion of the buffer. + The slice is valid for use only until the next buffer modification (that is, + only until the next call to a method like [Buffer.read], [Buffer.write], [Buffer.reset], or [Buffer.truncate]). + The slice aliases the buffer content at least until the next buffer modification, + so immediate changes to the slice will affect the result of future reads. + """ + return self.buf[self.off : len(self.buf)] + + # fn available_buffer(self) raises -> List[Byte]: + # """Returns an empty buffer with self.Available() capacity. + # This buffer is intended to be appended to and + # passed to an immediately succeeding [Buffer.write] call. + # The buffer is only valid until the next write operation on self. + # """ + # return self.buf[len(self.buf) :] + + fn __str__(self) -> String: + """Returns the contents of the unread portion of the buffer + as a string. If the [Buffer] is a nil pointer, it returns "". + + To build strings more efficiently, see the strings.Builder type. + + Creates a copy of the readable buffer and returns it as a string. + """ + var valid_bytes = self.buf[self.off : len(self.buf)] + + valid_bytes.append(0) + return String(valid_bytes) + + fn empty(self) -> Bool: + """Reports whether the unread portion of the buffer is empty.""" + return len(self.buf) <= self.off + + fn __len__(self) -> Int: + """Returns the number of bytes of the unread portion of the buffer; + self.buf.capacity == len(self.List[Byte]()).""" + return len(self.buf) - self.off + + fn cap(self) -> Int: + """Cap returns the capacity of the buffer's underlying byte slice, that is, the + total space allocated for the buffer's data.""" + return cap(self.buf) + + fn available(self) -> Int: + """Returns how many bytes are unused in the buffer.""" + return self.buf.capacity - len(self.buf) + + fn truncate(inout self, position: Int) raises: + """Discards all but the first n unread bytes from the buffer + but continues to use the same allocated storage. + It panics if position is negative or greater than the length of the buffer. + + Args: + position: The position to truncate the buffer to. + """ + if position == 0: + self.reset() + return + + self.last_read = OP_INVALID + if position < 0 or position > self.buf.capacity: + raise Error("buffer.Buffer: truncation out of range") + + self.buf = self.buf[: self.off + position] + + fn reset(inout self): + """Resets the buffer to be empty, + but it retains the underlying storage for use by future writes. + reset is the same as [buffer.truncate](0).""" + self.buf = List[Byte](capacity=self.buf.capacity) + self.off = 0 + self.last_read = OP_INVALID + + fn try_grow_by_reslice(inout self, n: Int) -> (Int, Bool): + """Inlineable version of grow for the fast-case where the + internal buffer only needs to be resliced. + It returns the index where bytes should be written and whether it succeeded.""" + var buffer_already_used = len(self.buf) + + if n <= self.buf.capacity - buffer_already_used: + # FIXME: It seems like reslicing in go can extend the length of the slice. Doens't work like that for my get slice impl. + # Instead, just add bytes of len(n) to the end of the buffer for now. + # self.buf = self.buf[: l + n] + self.buf.reserve(self.buf.capacity + n) + return buffer_already_used, True + + return 0, False + + fn grow(inout self, n: Int) -> Int: + """Grows the buffer to guarantee space for n more bytes. + It returns the index where bytes should be written. + If the buffer can't grow it will panic with ERR_TOO_LARGE.""" + var write_at: Int = len(self.buf) + # If buffer is empty, reset to recover space. + if write_at == 0 and self.off != 0: + self.reset() + + # Try to grow by means of a reslice. + var i: Int + var ok: Bool + i, ok = self.try_grow_by_reslice(n) + if ok: + return i + + # If buffer length is 0 and elements being added is less than small_buffer_size, resize the buffer and write from the beginning. + if self.buf.capacity == 0 and n <= SMALL_BUFFER_SIZE: + self.buf.reserve(SMALL_BUFFER_SIZE) + return 0 + + var c = cap(self.buf) + if Float64(n) <= c / 2 - write_at: + # We can slide things down instead of allocating a new + # slice. We only need m+n <= c to slide, but + # we instead var capacity get twice as large so we + # don't spend all our time copying. + _ = copy(self.buf, self.buf[self.off :]) + elif c > MAX_INT - c - n: + panic(ERR_TOO_LARGE) + # TODO: Commented out this branch because growing the slice here and then at the end is redundant? + # else: + # # Add self.off to account for self.buf[:self.off] being sliced off the front. + # # var sl = self.buf[self.off :] + # # self.buf = self.grow_slice(sl, self.off + n) + + # Restore self.off and len(self.buf). + self.off = 0 + # FIXME: It seems like reslicing in go can extend the length of the slice. Doens't work like that for my get slice impl. + # Instead, just add bytes of len(n) to the end of the buffer for now. + # self.buf = self.buf[: m + n] + self.buf.reserve(self.buf.capacity + n) + return write_at + + fn Grow(inout self, n: Int): + """Grows the buffer's capacity, if necessary, to guarantee space for + another n bytes. After grow(n), at least n bytes can be written to the + buffer without another allocation. + If n is negative, grow will panic. + If the buffer can't grow it will panic with [ERR_TOO_LARGE]. + """ + if n < 0: + panic("buffer.Buffer.Grow: negative count") + + var m = self.grow(n) + self.buf = self.buf[:m] + + fn write(inout self, src: List[Byte]) -> (Int, Error): + """Appends the contents of p to the buffer, growing the buffer as + needed. The return value n is the length of p; err is always nil. If the + buffer becomes too large, write will panic with [ERR_TOO_LARGE]. + + Args: + src: The bytes to write to the buffer. + + Returns: + The number of bytes written to the buffer. + """ + self.last_read = OP_INVALID + var write_at: Int + var ok: Bool + write_at, ok = self.try_grow_by_reslice(len(src)) + if not ok: + write_at = self.grow(len(src)) + + var bytes_written = copy(self.buf, src, write_at) + return bytes_written, Error() + + fn write_string(inout self, src: String) -> (Int, Error): + """Appends the contents of s to the buffer, growing the buffer as + needed. The return value n is the length of s; err is always nil. If the + buffer becomes too large, write_string will panic with [ERR_TOO_LARGE]. + + Args: + src: The bytes to write to the buffer. + + Returns: + The number of bytes written to the buffer. + """ + # self.last_read = OP_INVALID + # var write_at: Int + # var ok: Bool + # write_at, ok = self.try_grow_by_reslice(len(src)) + # if not ok: + # m = self.grow(len(src)) + # var b = self.buf[m:] + return self.write(src.as_bytes()) + + fn read_from[R: Reader](inout self, inout reader: R) -> (Int64, Error): + """Reads data from r until EOF and appends it to the buffer, growing + the buffer as needed. The return value n is the number of bytes read. Any + error except io.EOF encountered during the read is also returned. If the + buffer becomes too large, read_from will panic with [ERR_TOO_LARGE]. + + Args: + reader: The reader to read from. + + Returns: + The number of bytes read from the reader. + """ + self.last_read = OP_INVALID + var total_bytes_read: Int64 = 0 + while True: + _ = self.grow(MIN_READ) + + var bytes_read: Int + var err: Error + bytes_read, err = reader.read(self.buf) + if bytes_read < 0: + panic(ERR_NEGATIVE_READ) + + total_bytes_read += bytes_read + + var err_message = str(err) + if err_message != "": + if err_message == io.EOF: + return total_bytes_read, Error() + + return total_bytes_read, err + + fn grow_slice(self, inout b: List[Byte], n: Int) -> List[Byte]: + """Grows b by n, preserving the original content of self. + If the allocation fails, it panics with ERR_TOO_LARGE. + """ + # TODO(http:#golang.org/issue/51462): We should rely on the append-make + # pattern so that the compiler can call runtime.growslice. For example: + # return append(b, make(bytes, n)...) + # This avoids unnecessary zero-ing of the first b.capacity bytes of the + # allocated slice, but this pattern causes b to escape onto the heap. + # + # Instead use the append-make pattern with a nil slice to ensure that + # we allocate buffers rounded up to the closest size class. + var c = b.capacity + n # ensure enough space for n elements + if c < 2 * cap(b): + # The growth rate has historically always been 2x. In the future, + # we could rely purely on append to determine the growth rate. + c = 2 * cap(b) + + var resized_buffer = List[Byte](capacity=c) + _ = copy(resized_buffer, b) + # var b2: List[Byte] = List[Byte]() + # b2._vector.reserve(c) + + # # var b2 = append(bytes(nil), make(bytes, c)...) + # _ = copy(b2, b) + # return b2[:b.capacity] + # b._vector.reserve(c) + return resized_buffer[: b.capacity] + + fn write_to[W: Writer](inout self, inout writer: W) -> (Int64, Error): + """Writes data to w until the buffer is drained or an error occurs. + The return value n is the number of bytes written; it always fits into an + Int, but it is int64 to match the io.WriterTo trait. Any error + encountered during the write is also returned. + + Args: + writer: The writer to write to. + + Returns: + The number of bytes written to the writer. + """ + self.last_read = OP_INVALID + var bytes_to_write = len(self.buf) + var total_bytes_written: Int64 = 0 + + if bytes_to_write > 0: + # TODO: Replace usage of this intermeidate slice when normal slicing, once slice references work. + var sl = self.buf[self.off : bytes_to_write] + var bytes_written: Int + var err: Error + bytes_written, err = writer.write(sl) + if bytes_written > bytes_to_write: + panic("bytes.Buffer.write_to: invalid write count") + + self.off += bytes_written + total_bytes_written = Int64(bytes_written) + + var err_message = str(err) + if err_message != "": + return total_bytes_written, err + + # all bytes should have been written, by definition of write method in io.Writer + if bytes_written != bytes_to_write: + return total_bytes_written, Error(ERR_SHORT_WRITE) + + # Buffer is now empty; reset. + self.reset() + return total_bytes_written, Error() + + fn write_byte(inout self, byte: Byte) -> (Int, Error): + """Appends the byte c to the buffer, growing the buffer as needed. + The returned error is always nil, but is included to match [bufio.Writer]'s + write_byte. If the buffer becomes too large, write_byte will panic with + [ERR_TOO_LARGE]. + + Args: + byte: The byte to write to the buffer. + + Returns: + The number of bytes written to the buffer. + """ + self.last_read = OP_INVALID + var write_at: Int + var ok: Bool + write_at, ok = self.try_grow_by_reslice(1) + if not ok: + write_at = self.grow(1) + + _ = copy(self.buf, List[Byte](byte), write_at) + return write_at, Error() + + # fn write_rune(inout self, r: Rune) -> Int: + # """Appends the UTF-8 encoding of Unicode code point r to the + # buffer, returning its length and an error, which is always nil but is + # included to match [bufio.Writer]'s write_rune. The buffer is grown as needed; + # if it becomes too large, write_rune will panic with [ERR_TOO_LARGE]. + # """ + # # Compare as uint32 to correctly handle negative runes. + # if UInt32(r) < utf8.RuneSelf: + # self.write_byte(Byte(r)) + # return 1 + + # self.last_read = OP_INVALID + # var write_at: Int + # var ok: Bool + # write_at, ok = self.try_grow_by_reslice(utf8.UTFMax) + # if not ok: + # write_at = self.grow(utf8.UTFMax) + + # self.buf = utf8.AppendRune(self.buf[:write_at], r) + # return len(self.buf) - write_at + + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Reads the next len(dest) bytes from the buffer or until the buffer + is drained. The return value n is the number of bytes read. If the + buffer has no data to return, err is io.EOF (unless len(dest) is zero); + otherwise it is nil. + + Args: + dest: The buffer to read into. + + Returns: + The number of bytes read from the buffer. + """ + self.last_read = OP_INVALID + if self.empty(): + # Buffer is empty, reset to recover space. + self.reset() + if dest.capacity == 0: + return 0, Error() + return 0, Error(io.EOF) + + var bytes_read = copy(dest, self.buf[self.off : len(self.buf)]) + self.off += bytes_read + if bytes_read > 0: + self.last_read = OP_READ + + return bytes_read, Error() + + fn next(inout self, number_of_bytes: Int) raises -> List[Byte]: + """Returns a slice containing the next n bytes from the buffer, + advancing the buffer as if the bytes had been returned by [Buffer.read]. + If there are fewer than n bytes in the buffer, next returns the entire buffer. + The slice is only valid until the next call to a read or write method. + + Args: + number_of_bytes: The number of bytes to read from the buffer. + + Returns: + A slice containing the next n bytes from the buffer. + """ + self.last_read = OP_INVALID + var m = len(self) + var bytes_to_read = number_of_bytes + if bytes_to_read > m: + bytes_to_read = m + + var data = self.buf[self.off : self.off + bytes_to_read] + self.off += bytes_to_read + if bytes_to_read > 0: + self.last_read = OP_READ + + return data + + fn read_byte(inout self) -> (Byte, Error): + """Reads and returns the next byte from the buffer. + If no byte is available, it returns error io.EOF. + """ + if self.empty(): + # Buffer is empty, reset to recover space. + self.reset() + return Byte(0), Error(io.EOF) + + var byte = self.buf[self.off] + self.off += 1 + self.last_read = OP_READ + + return byte, Error() + + # read_rune reads and returns the next UTF-8-encoded + # Unicode code point from the buffer. + # If no bytes are available, the error returned is io.EOF. + # If the bytes are an erroneous UTF-8 encoding, it + # consumes one byte and returns U+FFFD, 1. + # fn read_rune(self) (r rune, size Int, err error) + # if self.empty() + # # Buffer is empty, reset to recover space. + # self.reset() + # return 0, 0, io.EOF + # + # c := self.buf[self.off] + # if c < utf8.RuneSelf + # self.off+= 1 + # self.last_read = OP_READ_RUNE1 + # return rune(c), 1, nil + # + # r, n := utf8.DecodeRune(self.buf[self.off:]) + # self.off += n + # self.last_read = ReadOp(n) + # return r, n, nil + # + + # unread_rune unreads the last rune returned by [Buffer.read_rune]. + # If the most recent read or write operation on the buffer was + # not a successful [Buffer.read_rune], unread_rune returns an error. (In this regard + # it is stricter than [Buffer.unread_byte], which will unread the last byte + # from any read operation.) + # fn unread_rune(self): + # if self.last_read <= OP_INVALID + # return errors.New("buffer.Buffer: unread_rune: previous operation was not a successful read_rune") + # + # if self.off >= Int(self.last_read) + # self.off -= Int(self.last_read) + # + # self.last_read = OP_INVALID + # return nil + + # var err_unread_byte = errors.New("buffer.Buffer: unread_byte: previous operation was not a successful read") + + fn unread_byte(inout self) -> Error: + """Unreads the last byte returned by the most recent successful + read operation that read at least one byte. If a write has happened since + the last read, if the last read returned an error, or if the read read zero + bytes, unread_byte returns an error. + """ + if self.last_read == OP_INVALID: + return Error("buffer.Buffer: unread_byte: previous operation was not a successful read") + + self.last_read = OP_INVALID + if self.off > 0: + self.off -= 1 + + return Error() + + fn read_bytes(inout self, delim: Byte) -> (List[Byte], Error): + """Reads until the first occurrence of delim in the input, + returning a slice containing the data up to and including the delimiter. + If read_bytes encounters an error before finding a delimiter, + it returns the data read before the error and the error itself (often io.EOF). + read_bytes returns err != nil if and only if the returned data does not end in + delim. + + Args: + delim: The delimiter to read until. + + Returns: + A List[Byte] struct containing the data up to and including the delimiter. + """ + var slice: List[Byte] + var err: Error + slice, err = self.read_slice(delim) + + # return a copy of slice. The buffer's backing array may + # be overwritten by later calls. + var line = List[Byte](capacity=BUFFER_SIZE) + for i in range(len(slice)): + line.append(slice[i]) + return line, Error() + + fn read_slice(inout self, delim: Byte) -> (List[Byte], Error): + """Like read_bytes but returns a reference to internal buffer data. + + Args: + delim: The delimiter to read until. + + Returns: + A List[Byte] struct containing the data up to and including the delimiter. + """ + var at_eof = False + var i = index_byte(self.buf[self.off : len(self.buf)], delim) + var end = self.off + i + 1 + + if i < 0: + end = len(self.buf) + at_eof = True + + var line = self.buf[self.off : end] + self.off = end + self.last_read = OP_READ + + if at_eof: + return line, Error(io.EOF) + + return line, Error() + + fn read_string(inout self, delim: Byte) -> (String, Error): + """Reads until the first occurrence of delim in the input, + returning a string containing the data up to and including the delimiter. + If read_string encounters an error before finding a delimiter, + it returns the data read before the error and the error itself (often io.EOF). + read_string returns err != nil if and only if the returned data does not end + in delim. + + Args: + delim: The delimiter to read until. + + Returns: + A string containing the data up to and including the delimiter. + """ + var slice: List[Byte] + var err: Error + slice, err = self.read_slice(delim) + slice.append(0) + return String(slice), err + + +fn new_buffer() -> Buffer: + """Creates and initializes a new [Buffer] using buf as its` + initial contents. The new [Buffer] takes ownership of buf, and the + caller should not use buf after this call. new_buffer is intended to + prepare a [Buffer] to read existing data. It can also be used to set + the initial size of the internal buffer for writing. To do that, + buf should have the desired capacity but a length of zero. + + In most cases, new([Buffer]) (or just declaring a [Buffer] variable) is + sufficient to initialize a [Buffer]. + """ + var b = List[Byte](capacity=BUFFER_SIZE) + return Buffer(b^) + + +fn new_buffer(owned buf: List[Byte]) -> Buffer: + """Creates and initializes a new [Buffer] using buf as its` + initial contents. The new [Buffer] takes ownership of buf, and the + caller should not use buf after this call. new_buffer is intended to + prepare a [Buffer] to read existing data. It can also be used to set + the initial size of the internal buffer for writing. To do that, + buf should have the desired capacity but a length of zero. + + In most cases, new([Buffer]) (or just declaring a [Buffer] variable) is + sufficient to initialize a [Buffer]. + + Args: + buf: The bytes to use as the initial contents of the buffer. + + Returns: + A new [Buffer] initialized with the provided bytes. + """ + return Buffer(buf^) + + +fn new_buffer(owned s: String) -> Buffer: + """Creates and initializes a new [Buffer] using string s as its + initial contents. It is intended to prepare a buffer to read an existing + string. + + In most cases, new([Buffer]) (or just declaring a [Buffer] variable) is + sufficient to initialize a [Buffer]. + + Args: + s: The string to use as the initial contents of the buffer. + + Returns: + A new [Buffer] initialized with the provided string. + """ + var bytes_buffer = List[Byte](s.as_bytes()) + return Buffer(bytes_buffer^) diff --git a/external/gojo/bytes/reader.mojo b/external/gojo/bytes/reader.mojo new file mode 100644 index 0000000..9539672 --- /dev/null +++ b/external/gojo/bytes/reader.mojo @@ -0,0 +1,216 @@ +from collections.optional import Optional +from ..builtins import cap, copy, Byte, panic +import ..io + + +@value +struct Reader( + Copyable, + Sized, + io.Reader, + io.ReaderAt, + io.WriterTo, + io.Seeker, + io.ByteReader, + io.ByteScanner, +): + """A Reader implements the io.Reader, io.ReaderAt, io.WriterTo, io.Seeker, + io.ByteScanner, and io.RuneScanner Interfaces by reading from + a byte slice. + Unlike a [Buffer], a Reader is read-only and supports seeking. + The zero value for Reader operates like a Reader of an empty slice. + """ + + var buffer: List[Byte] + var index: Int64 # current reading index + var prev_rune: Int # index of previous rune; or < 0 + + fn __len__(self) -> Int: + """len returns the number of bytes of the unread portion of the + slice.""" + if self.index >= len(self.buffer): + return 0 + + return int(len(self.buffer) - self.index) + + fn size(self) -> Int: + """Returns the original length of the underlying byte slice. + Size is the number of bytes available for reading via [Reader.ReadAt]. + The result is unaffected by any method calls except [Reader.Reset].""" + return len(self.buffer) + + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Reads from the internal buffer into the dest List[Byte] struct. + Implements the [io.Reader] Interface. + + Args: + dest: The destination List[Byte] struct to read into. + + Returns: + Int: The number of bytes read into dest.""" + if self.index >= len(self.buffer): + return 0, Error(io.EOF) + + self.prev_rune = -1 + var unread_bytes = self.buffer[int(self.index) : len(self.buffer)] + var bytes_read = copy(dest, unread_bytes) + + self.index += bytes_read + return bytes_read, Error() + + fn read_at(self, inout dest: List[Byte], off: Int64) -> (Int, Error): + """Reads len(dest) bytes into dest beginning at byte offset off. + Implements the [io.ReaderAt] Interface. + + Args: + dest: The destination List[Byte] struct to read into. + off: The offset to start reading from. + + Returns: + Int: The number of bytes read into dest. + """ + # cannot modify state - see io.ReaderAt + if off < 0: + return 0, Error("bytes.Reader.read_at: negative offset") + + if off >= Int64(len(self.buffer)): + return 0, Error(io.EOF) + + var unread_bytes = self.buffer[int(off) : len(self.buffer)] + var bytes_written = copy(dest, unread_bytes) + if bytes_written < len(dest): + return 0, Error(io.EOF) + + return bytes_written, Error() + + fn read_byte(inout self) -> (Byte, Error): + """Reads and returns a single byte from the internal buffer. Implements the [io.ByteReader] Interface.""" + self.prev_rune = -1 + if self.index >= len(self.buffer): + return Int8(0), Error(io.EOF) + + var byte = self.buffer[int(self.index)] + self.index += 1 + return byte, Error() + + fn unread_byte(inout self) -> Error: + """Unreads the last byte read by moving the read position back by one. + Complements [Reader.read_byte] in implementing the [io.ByteScanner] Interface. + """ + if self.index <= 0: + return Error("bytes.Reader.unread_byte: at beginning of slice") + + self.prev_rune = -1 + self.index -= 1 + + return Error() + + # # read_rune implements the [io.RuneReader] Interface. + # fn read_rune(self) (ch rune, size Int, err error): + # if self.index >= Int64(len(self.buffer)): + # self.prev_rune = -1 + # return 0, 0, io.EOF + + # self.prev_rune = Int(self.index) + # if c := self.buffer[self.index]; c < utf8.RuneSelf: + # self.index+= 1 + # return rune(c), 1, nil + + # ch, size = utf8.DecodeRune(self.buffer[self.index:]) + # self.index += Int64(size) + # return + + # # unread_rune complements [Reader.read_rune] in implementing the [io.RuneScanner] Interface. + # fn unread_rune(self) error: + # if self.index <= 0: + # return errors.New("bytes.Reader.unread_rune: at beginning of slice") + + # if self.prev_rune < 0: + # return errors.New("bytes.Reader.unread_rune: previous operation was not read_rune") + + # self.index = Int64(self.prev_rune) + # self.prev_rune = -1 + # return nil + + fn seek(inout self, offset: Int64, whence: Int) -> (Int64, Error): + """Moves the read position to the specified offset from the specified whence. + Implements the [io.Seeker] Interface. + + Args: + offset: The offset to move to. + whence: The reference point for offset. + + Returns: + The new position in which the next read will start from. + """ + self.prev_rune = -1 + var position: Int64 = 0 + + if whence == io.SEEK_START: + position = offset + elif whence == io.SEEK_CURRENT: + position = self.index + offset + elif whence == io.SEEK_END: + position = len(self.buffer) + offset + else: + return Int64(0), Error("bytes.Reader.seek: invalid whence") + + if position < 0: + return Int64(0), Error("bytes.Reader.seek: negative position") + + self.index = position + return position, Error() + + fn write_to[W: io.Writer](inout self, inout writer: W) -> (Int64, Error): + """Writes data to w until the buffer is drained or an error occurs. + implements the [io.WriterTo] Interface. + + Args: + writer: The writer to write to. + """ + self.prev_rune = -1 + if self.index >= len(self.buffer): + return Int64(0), Error() + + var bytes = self.buffer[int(self.index) : len(self.buffer)] + var write_count: Int + var err: Error + write_count, err = writer.write(bytes) + if write_count > len(bytes): + panic("bytes.Reader.write_to: invalid Write count") + + self.index += write_count + if write_count != len(bytes): + return Int64(write_count), Error(io.ERR_SHORT_WRITE) + + return Int64(write_count), Error() + + fn reset(inout self, buffer: List[Byte]): + """Resets the [Reader.Reader] to be reading from b. + + Args: + buffer: The new buffer to read from. + """ + self.buffer = buffer + self.index = 0 + self.prev_rune = -1 + + +fn new_reader(buffer: List[Byte]) -> Reader: + """Returns a new [Reader.Reader] reading from b. + + Args: + buffer: The new buffer to read from. + + """ + return Reader(buffer, 0, -1) + + +fn new_reader(buffer: String) -> Reader: + """Returns a new [Reader.Reader] reading from b. + + Args: + buffer: The new buffer to read from. + + """ + return Reader(buffer.as_bytes(), 0, -1) diff --git a/external/gojo/fmt/__init__.mojo b/external/gojo/fmt/__init__.mojo index fe5652b..a4b04e3 100644 --- a/external/gojo/fmt/__init__.mojo +++ b/external/gojo/fmt/__init__.mojo @@ -1 +1 @@ -from .fmt import sprintf, printf +from .fmt import sprintf, printf, sprintf_str diff --git a/external/gojo/fmt/fmt.mojo b/external/gojo/fmt/fmt.mojo index fe25b17..a44cdf3 100644 --- a/external/gojo/fmt/fmt.mojo +++ b/external/gojo/fmt/fmt.mojo @@ -8,35 +8,41 @@ Boolean Integer %d base 10 +%q a single-quoted character literal. +%x base 16, with lower-case letters for a-f +%X base 16, with upper-case letters for A-F Floating-point and complex constituents: %f decimal point but no exponent, e.g. 123.456 String and slice of bytes (treated equivalently with these verbs): %s the uninterpreted bytes of the string or slice +%q a double-quoted string TODO: - Add support for more formatting options - Switch to buffered writing to avoid multiple string concatenations - Add support for width and precision formatting options +- Handle escaping for String's %q """ from utils.variant import Variant +from math import floor +from ..builtins import Byte - -alias Args = Variant[String, Int, Float64, Bool] +alias Args = Variant[String, Int, Float64, Bool, List[Byte]] fn replace_first(s: String, old: String, new: String) -> String: """Replace the first occurrence of a substring in a string. - Parameters: - s (str): The original string - old (str): The substring to be replaced - new (str): The new substring + Args: + s: The original string + old: The substring to be replaced + new: The new substring Returns: - String: The string with the first occurrence of the old substring replaced by the new one. + The string with the first occurrence of the old substring replaced by the new one. """ # Find the first occurrence of the old substring var index = s.find(old) @@ -49,49 +55,122 @@ fn replace_first(s: String, old: String, new: String) -> String: return s -fn format_string(s: String, arg: String) -> String: - return replace_first(s, String("%s"), arg) +fn find_first_verb(s: String, verbs: List[String]) -> String: + """Find the first occurrence of a verb in a string. + + Args: + s: The original string + verbs: The list of verbs to search for. + + Returns: + The verb to replace. + """ + var index = -1 + var verb: String = "" + + for v in verbs: + var i = s.find(v[]) + if i != -1 and (index == -1 or i < index): + index = i + verb = v[] + + return verb + + +alias BASE10_TO_BASE16 = List[String]("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f") + + +fn convert_base10_to_base16(value: Int) -> String: + """Converts a base 10 number to base 16. + + Args: + value: Base 10 number. + + Returns: + Base 16 number as a String. + """ + + var val: Float64 = 0.0 + var result: Float64 = value + var base16: String = "" + while result > 1: + var temp = result / 16 + var floor_result = floor(temp) + var remainder = temp - floor_result + result = floor_result + val = 16 * remainder + + base16 = BASE10_TO_BASE16[int(val)] + base16 + + return base16 -fn format_integer(s: String, arg: Int) -> String: - return replace_first(s, String("%d"), arg) +fn format_string(format: String, arg: String) -> String: + var verb = find_first_verb(format, List[String]("%s", "%q")) + var arg_to_place = arg + if verb == "%q": + arg_to_place = '"' + arg + '"' + return replace_first(format, String("%s"), arg) -fn format_float(s: String, arg: Float64) -> String: - return replace_first(s, String("%f"), arg) +fn format_bytes(format: String, arg: List[Byte]) -> String: + var argument = arg + if argument[-1] != 0: + argument.append(0) -fn format_boolean(s: String, arg: Bool) -> String: - var value: String = "" + return format_string(format, argument) + + +fn format_integer(format: String, arg: Int) -> String: + var verb = find_first_verb(format, List[String]("%x", "%X", "%d", "%q")) + var arg_to_place = String(arg) + if verb == "%x": + arg_to_place = String(convert_base10_to_base16(arg)).lower() + elif verb == "%X": + arg_to_place = String(convert_base10_to_base16(arg)).upper() + elif verb == "%q": + arg_to_place = "'" + String(arg) + "'" + + return replace_first(format, verb, arg_to_place) + + +fn format_float(format: String, arg: Float64) -> String: + return replace_first(format, String("%f"), arg) + + +fn format_boolean(format: String, arg: Bool) -> String: + var value: String = "False" if arg: value = "True" - else: - value = "False" - return replace_first(s, String("%t"), value) + return replace_first(format, String("%t"), value) + + +# If the number of arguments does not match the number of format specifiers +alias BadArgCount = "(BAD ARG COUNT)" -fn sprintf(formatting: String, *args: Args) raises -> String: +fn sprintf(formatting: String, *args: Args) -> String: var text = formatting - var formatter_count = formatting.count("%") + var raw_percent_count = formatting.count("%%") * 2 + var formatter_count = formatting.count("%") - raw_percent_count - if formatter_count > len(args): - raise Error("Not enough arguments for format string") - elif formatter_count < len(args): - raise Error("Too many arguments for format string") + if formatter_count != len(args): + return BadArgCount for i in range(len(args)): var argument = args[i] if argument.isa[String](): text = format_string(text, argument.get[String]()[]) + elif argument.isa[List[Byte]](): + text = format_bytes(text, argument.get[List[Byte]]()[]) elif argument.isa[Int](): text = format_integer(text, argument.get[Int]()[]) elif argument.isa[Float64](): text = format_float(text, argument.get[Float64]()[]) elif argument.isa[Bool](): text = format_boolean(text, argument.get[Bool]()[]) - else: - raise Error("Unknown for argument #" + String(i)) return text @@ -114,7 +193,8 @@ fn sprintf_str(formatting: String, args: List[String]) raises -> String: fn printf(formatting: String, *args: Args) raises: var text = formatting - var formatter_count = formatting.count("%") + var raw_percent_count = formatting.count("%%") * 2 + var formatter_count = formatting.count("%") - raw_percent_count if formatter_count > len(args): raise Error("Not enough arguments for format string") @@ -125,6 +205,8 @@ fn printf(formatting: String, *args: Args) raises: var argument = args[i] if argument.isa[String](): text = format_string(text, argument.get[String]()[]) + elif argument.isa[List[Byte]](): + text = format_bytes(text, argument.get[List[Byte]]()[]) elif argument.isa[Int](): text = format_integer(text, argument.get[Int]()[]) elif argument.isa[Float64](): diff --git a/external/gojo/net/__init__.mojo b/external/gojo/net/__init__.mojo new file mode 100644 index 0000000..2587673 --- /dev/null +++ b/external/gojo/net/__init__.mojo @@ -0,0 +1,4 @@ +"""Adapted from go's net package + +A good chunk of the leg work here came from the lightbug_http project! https://github.com/saviorand/lightbug_http/tree/main +""" diff --git a/external/gojo/net/address.mojo b/external/gojo/net/address.mojo new file mode 100644 index 0000000..01bf25f --- /dev/null +++ b/external/gojo/net/address.mojo @@ -0,0 +1,145 @@ +@value +struct NetworkType: + var value: String + + alias empty = NetworkType("") + alias tcp = NetworkType("tcp") + alias tcp4 = NetworkType("tcp4") + alias tcp6 = NetworkType("tcp6") + alias udp = NetworkType("udp") + alias udp4 = NetworkType("udp4") + alias udp6 = NetworkType("udp6") + alias ip = NetworkType("ip") + alias ip4 = NetworkType("ip4") + alias ip6 = NetworkType("ip6") + alias unix = NetworkType("unix") + + +trait Addr(CollectionElement, Stringable): + fn network(self) -> String: + """Name of the network (for example, "tcp", "udp").""" + ... + + +@value +struct TCPAddr(Addr): + """Addr struct representing a TCP address. + + Args: + ip: IP address. + port: Port number. + zone: IPv6 addressing zone. + """ + + var ip: String + var port: Int + var zone: String # IPv6 addressing zone + + fn __init__(inout self): + self.ip = String("127.0.0.1") + self.port = 8000 + self.zone = "" + + fn __init__(inout self, ip: String, port: Int): + self.ip = ip + self.port = port + self.zone = "" + + fn __str__(self) -> String: + if self.zone != "": + return join_host_port(String(self.ip) + "%" + self.zone, self.port) + return join_host_port(self.ip, self.port) + + fn network(self) -> String: + return NetworkType.tcp.value + + +fn resolve_internet_addr(network: String, address: String) raises -> TCPAddr: + var host: String = "" + var port: String = "" + var portnum: Int = 0 + if ( + network == NetworkType.tcp.value + or network == NetworkType.tcp4.value + or network == NetworkType.tcp6.value + or network == NetworkType.udp.value + or network == NetworkType.udp4.value + or network == NetworkType.udp6.value + ): + if address != "": + var host_port = split_host_port(address) + host = host_port.host + port = host_port.port + portnum = atol(port.__str__()) + elif network == NetworkType.ip.value or network == NetworkType.ip4.value or network == NetworkType.ip6.value: + if address != "": + host = address + elif network == NetworkType.unix.value: + raise Error("Unix addresses not supported yet") + else: + raise Error("unsupported network type: " + network) + return TCPAddr(host, portnum) + + +alias missingPortError = Error("missing port in address") +alias tooManyColonsError = Error("too many colons in address") + + +struct HostPort(Stringable): + var host: String + var port: Int + + fn __init__(inout self, host: String, port: Int): + self.host = host + self.port = port + + fn __str__(self) -> String: + return join_host_port(self.host, str(self.port)) + + +fn join_host_port(host: String, port: String) -> String: + if host.find(":") != -1: # must be IPv6 literal + return "[" + host + "]:" + port + return host + ":" + port + + +fn split_host_port(hostport: String) raises -> HostPort: + var host: String = "" + var port: String = "" + var colon_index = hostport.rfind(":") + var j: Int = 0 + var k: Int = 0 + + if colon_index == -1: + raise missingPortError + if hostport[0] == "[": + var end_bracket_index = hostport.find("]") + if end_bracket_index == -1: + raise Error("missing ']' in address") + if end_bracket_index + 1 == len(hostport): + raise missingPortError + elif end_bracket_index + 1 == colon_index: + host = hostport[1:end_bracket_index] + j = 1 + k = end_bracket_index + 1 + else: + if hostport[end_bracket_index + 1] == ":": + raise tooManyColonsError + else: + raise missingPortError + else: + host = hostport[:colon_index] + if host.find(":") != -1: + raise tooManyColonsError + if hostport[j:].find("[") != -1: + raise Error("unexpected '[' in address") + if hostport[k:].find("]") != -1: + raise Error("unexpected ']' in address") + port = hostport[colon_index + 1 :] + + if port == "": + raise missingPortError + if host == "": + raise Error("missing host") + + return HostPort(host, atol(port)) diff --git a/external/gojo/net/dial.mojo b/external/gojo/net/dial.mojo new file mode 100644 index 0000000..5effd65 --- /dev/null +++ b/external/gojo/net/dial.mojo @@ -0,0 +1,45 @@ +from .tcp import TCPAddr, TCPConnection, resolve_internet_addr +from .socket import Socket +from .address import split_host_port + + +@value +struct Dialer: + var local_address: TCPAddr + + fn dial(self, network: String, address: String) raises -> TCPConnection: + var tcp_addr = resolve_internet_addr(network, address) + var socket = Socket(local_address=self.local_address) + socket.connect(tcp_addr.ip, tcp_addr.port) + print(String("Connected to ") + socket.remote_address) + return TCPConnection(socket^) + + +fn dial_tcp(network: String, remote_address: TCPAddr) raises -> TCPConnection: + """Connects to the address on the named network. + + The network must be "tcp", "tcp4", or "tcp6". + Args: + network: The network type. + remote_address: The remote address to connect to. + + Returns: + The TCP connection. + """ + # TODO: Add conversion of domain name to ip address + return Dialer(remote_address).dial(network, remote_address.ip + ":" + str(remote_address.port)) + + +fn dial_tcp(network: String, remote_address: String) raises -> TCPConnection: + """Connects to the address on the named network. + + The network must be "tcp", "tcp4", or "tcp6". + Args: + network: The network type. + remote_address: The remote address to connect to. + + Returns: + The TCP connection. + """ + var address = split_host_port(remote_address) + return Dialer(TCPAddr(address.host, address.port)).dial(network, remote_address) diff --git a/external/gojo/net/fd.mojo b/external/gojo/net/fd.mojo new file mode 100644 index 0000000..6e4fc62 --- /dev/null +++ b/external/gojo/net/fd.mojo @@ -0,0 +1,77 @@ +from collections.optional import Optional +import ..io +from ..builtins import Byte +from ..syscall.file import close +from ..syscall.types import c_char +from ..syscall.net import ( + recv, + send, + strlen, +) + +alias O_RDWR = 0o2 + + +trait FileDescriptorBase(io.Reader, io.Writer, io.Closer): + ... + + +struct FileDescriptor(FileDescriptorBase): + var fd: Int + var is_closed: Bool + + # This takes ownership of a POSIX file descriptor. + fn __moveinit__(inout self, owned existing: Self): + self.fd = existing.fd + self.is_closed = existing.is_closed + + fn __init__(inout self, fd: Int): + self.fd = fd + self.is_closed = False + + fn __del__(owned self): + if not self.is_closed: + var err = self.close() + if err: + print(str(err)) + + fn close(inout self) -> Error: + """Mark the file descriptor as closed.""" + var close_status = close(self.fd) + if close_status == -1: + return Error("FileDescriptor.close: Failed to close socket") + + self.is_closed = True + return Error() + + fn dup(self) -> Self: + """Duplicate the file descriptor.""" + var new_fd = external_call["dup", Int, Int](self.fd) + return Self(new_fd) + + # TODO: Need faster approach to copying data from the file descriptor to the buffer. + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Receive data from the file descriptor and write it to the buffer provided.""" + var ptr = Pointer[UInt8]().alloc(dest.capacity) + var bytes_received = recv(self.fd, ptr, dest.capacity, 0) + if bytes_received == -1: + return 0, Error("Failed to receive message from socket.") + + var int8_ptr = ptr.bitcast[Int8]() + for i in range(bytes_received): + dest.append(int8_ptr[i]) + + if bytes_received < dest.capacity: + return bytes_received, Error(io.EOF) + + return bytes_received, Error() + + fn write(inout self, src: List[Byte]) -> (Int, Error): + """Write data from the buffer to the file descriptor.""" + var header_pointer = Pointer[Int8](src.data.address).bitcast[UInt8]() + + var bytes_sent = send(self.fd, header_pointer, strlen(header_pointer), 0) + if bytes_sent == -1: + return 0, Error("Failed to send message") + + return bytes_sent, Error() diff --git a/external/gojo/net/ip.mojo b/external/gojo/net/ip.mojo new file mode 100644 index 0000000..76a56bd --- /dev/null +++ b/external/gojo/net/ip.mojo @@ -0,0 +1,178 @@ +from utils.variant import Variant +from sys.info import os_is_linux, os_is_macos +from ..syscall.types import ( + c_int, + c_char, + c_void, + c_uint, +) +from ..syscall.net import ( + addrinfo, + addrinfo_unix, + AF_INET, + SOCK_STREAM, + AI_PASSIVE, + sockaddr, + sockaddr_in, + htons, + ntohs, + inet_pton, + inet_ntop, + getaddrinfo, + getaddrinfo_unix, + gai_strerror, + to_char_ptr, + c_charptr_to_string, +) + +alias AddrInfo = Variant[addrinfo, addrinfo_unix] + + +fn get_addr_info(host: String) raises -> AddrInfo: + var status: Int32 = 0 + if os_is_macos(): + var servinfo = Pointer[addrinfo]().alloc(1) + servinfo.store(addrinfo()) + var hints = addrinfo() + hints.ai_family = AF_INET + hints.ai_socktype = SOCK_STREAM + hints.ai_flags = AI_PASSIVE + + var host_ptr = to_char_ptr(host) + + var status = getaddrinfo( + host_ptr, + Pointer[UInt8](), + Pointer.address_of(hints), + Pointer.address_of(servinfo), + ) + if status != 0: + print("getaddrinfo failed to execute with status:", status) + var msg_ptr = gai_strerror(c_int(status)) + _ = external_call["printf", c_int, Pointer[c_char], Pointer[c_char]]( + to_char_ptr("gai_strerror: %s"), msg_ptr + ) + var msg = c_charptr_to_string(msg_ptr) + print("getaddrinfo error message: ", msg) + + if not servinfo: + print("servinfo is null") + raise Error("Failed to get address info. Pointer to addrinfo is null.") + + return servinfo.load() + elif os_is_linux(): + var servinfo = Pointer[addrinfo_unix]().alloc(1) + servinfo.store(addrinfo_unix()) + var hints = addrinfo_unix() + hints.ai_family = AF_INET + hints.ai_socktype = SOCK_STREAM + hints.ai_flags = AI_PASSIVE + + var host_ptr = to_char_ptr(host) + + var status = getaddrinfo_unix( + host_ptr, + Pointer[UInt8](), + Pointer.address_of(hints), + Pointer.address_of(servinfo), + ) + if status != 0: + print("getaddrinfo failed to execute with status:", status) + var msg_ptr = gai_strerror(c_int(status)) + _ = external_call["printf", c_int, Pointer[c_char], Pointer[c_char]]( + to_char_ptr("gai_strerror: %s"), msg_ptr + ) + var msg = c_charptr_to_string(msg_ptr) + print("getaddrinfo error message: ", msg) + + if not servinfo: + print("servinfo is null") + raise Error("Failed to get address info. Pointer to addrinfo is null.") + + return servinfo.load() + else: + raise Error("Windows is not supported yet! Sorry!") + + +fn get_ip_address(host: String) raises -> String: + """Get the IP address of a host.""" + # Call getaddrinfo to get the IP address of the host. + var result = get_addr_info(host) + var ai_addr: Pointer[sockaddr] + var address_family: Int32 = 0 + var address_length: UInt32 = 0 + if result.isa[addrinfo](): + var addrinfo = result.get[addrinfo]() + ai_addr = addrinfo[].ai_addr + address_family = addrinfo[].ai_family + address_length = addrinfo[].ai_addrlen + else: + var addrinfo = result.get[addrinfo_unix]() + ai_addr = addrinfo[].ai_addr + address_family = addrinfo[].ai_family + address_length = addrinfo[].ai_addrlen + + if not ai_addr: + print("ai_addr is null") + raise Error("Failed to get IP address. getaddrinfo was called successfully, but ai_addr is null.") + + # Cast sockaddr struct to sockaddr_in struct and convert the binary IP to a string using inet_ntop. + var addr_in = ai_addr.bitcast[sockaddr_in]().load() + + return convert_binary_ip_to_string(addr_in.sin_addr.s_addr, address_family, address_length).strip() + + +fn convert_port_to_binary(port: Int) -> UInt16: + return htons(UInt16(port)) + + +fn convert_binary_port_to_int(port: UInt16) -> Int: + return int(ntohs(port)) + + +fn convert_ip_to_binary(ip_address: String, address_family: Int) -> UInt32: + var ip_buffer = Pointer[c_void].alloc(4) + var status = inet_pton(address_family, to_char_ptr(ip_address), ip_buffer) + if status == -1: + print("Failed to convert IP address to binary") + + return ip_buffer.bitcast[c_uint]().load() + + +fn convert_binary_ip_to_string(owned ip_address: UInt32, address_family: Int32, address_length: UInt32) -> String: + """Convert a binary IP address to a string by calling inet_ntop. + + Args: + ip_address: The binary IP address. + address_family: The address family of the IP address. + address_length: The length of the address. + + Returns: + The IP address as a string. + """ + # It seems like the len of the buffer depends on the length of the string IP. + # Allocating 10 works for localhost (127.0.0.1) which I suspect is 9 bytes + 1 null terminator byte. So max should be 16 (15 + 1). + var ip_buffer = Pointer[c_void].alloc(16) + var ip_address_ptr = Pointer.address_of(ip_address).bitcast[c_void]() + _ = inet_ntop(address_family, ip_address_ptr, ip_buffer, 16) + + var string_buf = ip_buffer.bitcast[Int8]() + var index = 0 + while True: + if string_buf[index] == 0: + break + index += 1 + + return StringRef(string_buf, index) + + +fn build_sockaddr_pointer(ip_address: String, port: Int, address_family: Int) -> Pointer[sockaddr]: + """Build a sockaddr pointer from an IP address and port number. + https://learn.microsoft.com/en-us/windows/win32/winsock/sockaddr-2 + https://learn.microsoft.com/en-us/windows/win32/api/ws2def/ns-ws2def-sockaddr_in. + """ + var bin_port = convert_port_to_binary(port) + var bin_ip = convert_ip_to_binary(ip_address, address_family) + + var ai = sockaddr_in(address_family, bin_port, bin_ip, StaticTuple[c_char, 8]()) + return Pointer[sockaddr_in].address_of(ai).bitcast[sockaddr]() diff --git a/external/gojo/net/net.mojo b/external/gojo/net/net.mojo new file mode 100644 index 0000000..324681a --- /dev/null +++ b/external/gojo/net/net.mojo @@ -0,0 +1,130 @@ +from memory._arc import Arc +import ..io +from ..builtins import Byte +from .socket import Socket +from .address import Addr, TCPAddr + +alias DEFAULT_BUFFER_SIZE = 4096 + + +trait Conn(io.Writer, io.Reader, io.Closer): + fn __init__(inout self, owned socket: Socket): + ... + + """Conn is a generic stream-oriented network connection.""" + + fn local_address(self) -> TCPAddr: + """Returns the local network address, if known.""" + ... + + fn remote_address(self) -> TCPAddr: + """Returns the local network address, if known.""" + ... + + # fn set_deadline(self, t: time.Time) -> Error: + # """Sets the read and write deadlines associated + # with the connection. It is equivalent to calling both + # SetReadDeadline and SetWriteDeadline. + + # A deadline is an absolute time after which I/O operations + # fail instead of blocking. The deadline applies to all future + # and pending I/O, not just the immediately following call to + # read or write. After a deadline has been exceeded, the + # connection can be refreshed by setting a deadline in the future. + + # If the deadline is exceeded a call to read or write or to other + # I/O methods will return an error that wraps os.ErrDeadlineExceeded. + # This can be tested using errors.Is(err, os.ErrDeadlineExceeded). + # The error's Timeout method will return true, but note that there + # are other possible errors for which the Timeout method will + # return true even if the deadline has not been exceeded. + + # An idle timeout can be implemented by repeatedly extending + # the deadline after successful read or write calls. + + # A zero value for t means I/O operations will not time out.""" + # ... + + # fn set_read_deadline(self, t: time.Time) -> Error: + # """Sets the deadline for future read calls + # and any currently-blocked read call. + # A zero value for t means read will not time out.""" + # ... + + # fn set_write_deadline(self, t: time.Time) -> Error: + # """Sets the deadline for future write calls + # and any currently-blocked write call. + # Even if write times out, it may return n > 0, indicating that + # some of the data was successfully written. + # A zero value for t means write will not time out.""" + # ... + + +@value +struct Connection(Conn): + """Connection is a concrete generic stream-oriented network connection. + It is used as the internal connection for structs like TCPConnection. + + Args: + fd: The file descriptor of the connection. + """ + + var fd: Arc[Socket] + + fn __init__(inout self, owned socket: Socket): + self.fd = Arc(socket^) + + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Reads data from the underlying file descriptor. + + Args: + dest: The buffer to read data into. + + Returns: + The number of bytes read, or an error if one occurred. + """ + var bytes_written: Int = 0 + var err = Error() + bytes_written, err = self.fd[].read(dest) + if err: + if str(err) != io.EOF: + return 0, err + + return bytes_written, err + + fn write(inout self, src: List[Byte]) -> (Int, Error): + """Writes data to the underlying file descriptor. + + Args: + src: The buffer to read data into. + + Returns: + The number of bytes written, or an error if one occurred. + """ + var bytes_read: Int = 0 + var err = Error() + bytes_read, err = self.fd[].write(src) + if err: + return 0, err + + return bytes_read, err + + fn close(inout self) -> Error: + """Closes the underlying file descriptor. + + Returns: + An error if one occurred, or None if the file descriptor was closed successfully. + """ + return self.fd[].close() + + fn local_address(self) -> TCPAddr: + """Returns the local network address. + The Addr returned is shared by all invocations of local_address, so do not modify it. + """ + return self.fd[].local_address + + fn remote_address(self) -> TCPAddr: + """Returns the remote network address. + The Addr returned is shared by all invocations of remote_address, so do not modify it. + """ + return self.fd[].remote_address diff --git a/external/gojo/net/socket.mojo b/external/gojo/net/socket.mojo new file mode 100644 index 0000000..594cc23 --- /dev/null +++ b/external/gojo/net/socket.mojo @@ -0,0 +1,432 @@ +from collections.optional import Optional +from ..builtins import Byte +from ..syscall.file import close +from ..syscall.types import ( + c_void, + c_uint, + c_char, + c_int, +) +from ..syscall.net import ( + sockaddr, + sockaddr_in, + addrinfo, + addrinfo_unix, + socklen_t, + socket, + connect, + recv, + send, + shutdown, + inet_pton, + inet_ntoa, + inet_ntop, + to_char_ptr, + htons, + ntohs, + strlen, + getaddrinfo, + getaddrinfo_unix, + gai_strerror, + c_charptr_to_string, + bind, + listen, + accept, + setsockopt, + getsockopt, + getsockname, + getpeername, + AF_INET, + SOCK_STREAM, + SHUT_RDWR, + AI_PASSIVE, + SOL_SOCKET, + SO_REUSEADDR, + SO_RCVTIMEO, +) +from .fd import FileDescriptor, FileDescriptorBase +from .ip import ( + convert_binary_ip_to_string, + build_sockaddr_pointer, + convert_binary_port_to_int, +) +from .address import Addr, TCPAddr, HostPort + +alias SocketClosedError = Error("Socket: Socket is already closed") + + +struct Socket(FileDescriptorBase): + """Represents a network file descriptor. Wraps around a file descriptor and provides network functions. + + Args: + local_address: The local address of the socket (local address if bound). + remote_address: The remote address of the socket (peer's address if connected). + address_family: The address family of the socket. + socket_type: The socket type. + protocol: The protocol. + """ + + var sockfd: FileDescriptor + var address_family: Int + var socket_type: UInt8 + var protocol: UInt8 + var local_address: TCPAddr + var remote_address: TCPAddr + var _closed: Bool + var _is_connected: Bool + + fn __init__( + inout self, + local_address: TCPAddr = TCPAddr(), + remote_address: TCPAddr = TCPAddr(), + address_family: Int = AF_INET, + socket_type: UInt8 = SOCK_STREAM, + protocol: UInt8 = 0, + ) raises: + """Create a new socket object. + + Args: + local_address: The local address of the socket (local address if bound). + remote_address: The remote address of the socket (peer's address if connected). + address_family: The address family of the socket. + socket_type: The socket type. + protocol: The protocol. + """ + self.address_family = address_family + self.socket_type = socket_type + self.protocol = protocol + + var fd = socket(address_family, SOCK_STREAM, 0) + if fd == -1: + raise Error("Socket creation error") + self.sockfd = FileDescriptor(int(fd)) + self.local_address = local_address + self.remote_address = remote_address + self._closed = False + self._is_connected = False + + fn __init__( + inout self, + fd: Int32, + address_family: Int, + socket_type: UInt8, + protocol: UInt8, + local_address: TCPAddr = TCPAddr(), + remote_address: TCPAddr = TCPAddr(), + ): + """ + Create a new socket object when you already have a socket file descriptor. Typically through socket.accept(). + + Args: + fd: The file descriptor of the socket. + address_family: The address family of the socket. + socket_type: The socket type. + protocol: The protocol. + local_address: Local address of socket. + remote_address: Remote address of port. + """ + self.sockfd = FileDescriptor(int(fd)) + self.address_family = address_family + self.socket_type = socket_type + self.protocol = protocol + self.local_address = local_address + self.remote_address = remote_address + self._closed = False + self._is_connected = True + + fn __moveinit__(inout self, owned existing: Self): + self.sockfd = existing.sockfd^ + self.address_family = existing.address_family + self.socket_type = existing.socket_type + self.protocol = existing.protocol + self.local_address = existing.local_address^ + self.remote_address = existing.remote_address^ + self._closed = existing._closed + self._is_connected = existing._is_connected + + # fn __enter__(self) -> Self: + # return self + + # fn __exit__(inout self) raises: + # if self._is_connected: + # self.shutdown() + # if not self._closed: + # self.close() + + fn __del__(owned self): + if self._is_connected: + self.shutdown() + if not self._closed: + var err = self.close() + _ = self.sockfd.fd + if err: + print("Failed to close socket during deletion:", str(err)) + + @always_inline + fn accept(self) raises -> Self: + """Accept a connection. The socket must be bound to an address and listening for connections. + The return value is a connection where conn is a new socket object usable to send and receive data on the connection, + and address is the address bound to the socket on the other end of the connection. + """ + var their_addr_ptr = Pointer[sockaddr].alloc(1) + var sin_size = socklen_t(sizeof[socklen_t]()) + var new_sockfd = accept(self.sockfd.fd, their_addr_ptr, Pointer[socklen_t].address_of(sin_size)) + if new_sockfd == -1: + raise Error("Failed to accept connection") + + var remote = self.get_peer_name() + return Self( + new_sockfd, + self.address_family, + self.socket_type, + self.protocol, + self.local_address, + TCPAddr(remote.host, remote.port), + ) + + fn listen(self, backlog: Int = 0) raises: + """Enable a server to accept connections. + + Args: + backlog: The maximum number of queued connections. Should be at least 0, and the maximum is system-dependent (usually 5). + """ + var queued = backlog + if backlog < 0: + queued = 0 + if listen(self.sockfd.fd, queued) == -1: + raise Error("Failed to listen for connections") + + @always_inline + fn bind(inout self, address: String, port: Int) raises: + """Bind the socket to address. The socket must not already be bound. (The format of address depends on the address family). + + When a socket is created with Socket(), it exists in a name + space (address family) but has no address assigned to it. bind() + assigns the address specified by addr to the socket referred to + by the file descriptor sockfd. addrlen specifies the size, in + bytes, of the address structure pointed to by addr. + Traditionally, this operation is called 'assigning a name to a + socket'. + + Args: + address: String - The IP address to bind the socket to. + port: The port number to bind the socket to. + """ + var sockaddr_pointer = build_sockaddr_pointer(address, port, self.address_family) + + if bind(self.sockfd.fd, sockaddr_pointer, sizeof[sockaddr_in]()) == -1: + _ = shutdown(self.sockfd.fd, SHUT_RDWR) + raise Error("Binding socket failed. Wait a few seconds and try again?") + + var local = self.get_sock_name() + self.local_address = TCPAddr(local.host, local.port) + + @always_inline + fn file_no(self) -> Int32: + """Return the file descriptor of the socket.""" + return self.sockfd.fd + + @always_inline + fn get_sock_name(self) raises -> HostPort: + """Return the address of the socket.""" + if self._closed: + raise SocketClosedError + + # TODO: Add check to see if the socket is bound and error if not. + + var local_address_ptr = Pointer[sockaddr].alloc(1) + var local_address_ptr_size = socklen_t(sizeof[sockaddr]()) + var status = getsockname( + self.sockfd.fd, + local_address_ptr, + Pointer[socklen_t].address_of(local_address_ptr_size), + ) + if status == -1: + raise Error("Socket.get_sock_name: Failed to get address of local socket.") + var addr_in = local_address_ptr.bitcast[sockaddr_in]().load() + + return HostPort( + host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AF_INET, 16), + port=convert_binary_port_to_int(addr_in.sin_port), + ) + + fn get_peer_name(self) raises -> HostPort: + """Return the address of the peer connected to the socket.""" + if self._closed: + raise SocketClosedError + + # TODO: Add check to see if the socket is bound and error if not. + var remote_address_ptr = Pointer[sockaddr].alloc(1) + var remote_address_ptr_size = socklen_t(sizeof[sockaddr]()) + var status = getpeername( + self.sockfd.fd, + remote_address_ptr, + Pointer[socklen_t].address_of(remote_address_ptr_size), + ) + if status == -1: + raise Error("Socket.get_peer_name: Failed to get address of remote socket.") + + # Cast sockaddr struct to sockaddr_in to convert binary IP to string. + var addr_in = remote_address_ptr.bitcast[sockaddr_in]().load() + + return HostPort( + host=convert_binary_ip_to_string(addr_in.sin_addr.s_addr, AF_INET, 16), + port=convert_binary_port_to_int(addr_in.sin_port), + ) + + fn get_socket_option(self, option_name: Int) raises -> Int: + """Return the value of the given socket option. + + Args: + option_name: The socket option to get. + """ + var option_value_pointer = Pointer[c_void].alloc(1) + var option_len = socklen_t(sizeof[socklen_t]()) + var option_len_pointer = Pointer.address_of(option_len) + var status = getsockopt( + self.sockfd.fd, + SOL_SOCKET, + option_name, + option_value_pointer, + option_len_pointer, + ) + if status == -1: + raise Error("Socket.get_sock_opt failed with status: " + str(status)) + + return option_value_pointer.bitcast[Int]().load() + + fn set_socket_option(self, option_name: Int, owned option_value: UInt8 = 1) raises: + """Return the value of the given socket option. + + Args: + option_name: The socket option to set. + option_value: The value to set the socket option to. + """ + var option_value_pointer = Pointer[c_void].address_of(option_value) + var option_len = sizeof[socklen_t]() + var status = setsockopt(self.sockfd.fd, SOL_SOCKET, option_name, option_value_pointer, option_len) + if status == -1: + raise Error("Socket.set_sock_opt failed with status: " + str(status)) + + fn connect(inout self, address: String, port: Int) raises: + """Connect to a remote socket at address. + + Args: + address: String - The IP address to connect to. + port: The port number to connect to. + """ + var sockaddr_pointer = build_sockaddr_pointer(address, port, self.address_family) + + if connect(self.sockfd.fd, sockaddr_pointer, sizeof[sockaddr_in]()) == -1: + self.shutdown() + raise Error("Socket.connect: Failed to connect to the remote socket at: " + address + ":" + str(port)) + + var remote = self.get_peer_name() + self.remote_address = TCPAddr(remote.host, remote.port) + + fn write(inout self: Self, src: List[Byte]) -> (Int, Error): + """Send data to the socket. The socket must be connected to a remote socket. + + Args: + src: The data to send. + + Returns: + The number of bytes sent. + """ + var bytes_written: Int + var err: Error + bytes_written, err = self.sockfd.write(src) + if err: + return 0, err + + return bytes_written, Error() + + fn send_all(self, src: List[Byte], max_attempts: Int = 3) raises: + """Send data to the socket. The socket must be connected to a remote socket. + + Args: + src: The data to send. + max_attempts: The maximum number of attempts to send the data. + """ + var header_pointer = Pointer[Int8](src.data.address).bitcast[UInt8]() + var total_bytes_sent = 0 + var attempts = 0 + + # Try to send all the data in the buffer. If it did not send all the data, keep trying but start from the offset of the last successful send. + while total_bytes_sent < len(src): + if attempts > max_attempts: + raise Error("Failed to send message after " + String(max_attempts) + " attempts.") + + var bytes_sent = send( + self.sockfd.fd, + header_pointer.offset(total_bytes_sent), + strlen(header_pointer.offset(total_bytes_sent)), + 0, + ) + if bytes_sent == -1: + raise Error("Failed to send message, wrote" + String(total_bytes_sent) + "bytes before failing.") + total_bytes_sent += bytes_sent + attempts += 1 + + fn send_to(inout self, src: List[Byte], address: String, port: Int) raises -> Int: + """Send data to the a remote address by connecting to the remote socket before sending. + The socket must be not already be connected to a remote socket. + + Args: + src: The data to send. + address: The IP address to connect to. + port: The port number to connect to. + """ + var header_pointer = Pointer[Int8](src.data.address).bitcast[UInt8]() + self.connect(address, port) + var bytes_written: Int + var err: Error + bytes_written, err = self.write(src) + if err: + raise err + return bytes_written + + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Receive data from the socket.""" + # Not ideal since we can't use the pointer from the List[Byte] struct directly. So we use a temporary pointer to receive the data. + # Then we copy all the data over. + var bytes_written: Int + var err: Error + bytes_written, err = self.sockfd.read(dest) + if err: + if str(err) != "EOF": + return 0, err + + return bytes_written, Error() + + fn shutdown(self): + _ = shutdown(self.sockfd.fd, SHUT_RDWR) + + fn close(inout self) -> Error: + """Mark the socket closed. + Once that happens, all future operations on the socket object will fail. + The remote end will receive no more data (after queued data is flushed). + """ + self.shutdown() + var err = self.sockfd.close() + if err: + return err + + self._closed = True + return Error() + + # TODO: Trying to set timeout fails, but some other options don't? + # fn get_timeout(self) raises -> Seconds: + # """Return the timeout value for the socket.""" + # return self.get_socket_option(SO_RCVTIMEO) + + # fn set_timeout(self, owned duration: Seconds) raises: + # """Set the timeout value for the socket. + + # Args: + # duration: Seconds - The timeout duration in seconds. + # """ + # self.set_socket_option(SO_RCVTIMEO, duration) + + fn send_file(self, file: FileHandle, offset: Int = 0) raises: + self.send_all(file.read_bytes()) diff --git a/external/gojo/net/tcp.mojo b/external/gojo/net/tcp.mojo new file mode 100644 index 0000000..6a59db8 --- /dev/null +++ b/external/gojo/net/tcp.mojo @@ -0,0 +1,207 @@ +from ..builtins import Byte +from ..syscall.net import SO_REUSEADDR +from .net import Connection, Conn +from .address import TCPAddr, NetworkType, split_host_port +from .socket import Socket + + +# Time in nanoseconds +alias Duration = Int +alias DEFAULT_BUFFER_SIZE = 4096 +alias DEFAULT_TCP_KEEP_ALIVE = Duration(15 * 1000 * 1000 * 1000) # 15 seconds + + +fn resolve_internet_addr(network: String, address: String) raises -> TCPAddr: + var host: String = "" + var port: String = "" + var portnum: Int = 0 + if ( + network == NetworkType.tcp.value + or network == NetworkType.tcp4.value + or network == NetworkType.tcp6.value + or network == NetworkType.udp.value + or network == NetworkType.udp4.value + or network == NetworkType.udp6.value + ): + if address != "": + var host_port = split_host_port(address) + host = host_port.host + port = host_port.port + portnum = atol(port.__str__()) + elif network == NetworkType.ip.value or network == NetworkType.ip4.value or network == NetworkType.ip6.value: + if address != "": + host = address + elif network == NetworkType.unix.value: + raise Error("Unix addresses not supported yet") + else: + raise Error("unsupported network type: " + network) + return TCPAddr(host, portnum) + + +# TODO: For now listener is paired with TCP until we need to support +# more than one type of Connection or Listener +@value +struct ListenConfig(CollectionElement): + var keep_alive: Duration + + fn listen(self, network: String, address: String) raises -> TCPListener: + var tcp_addr = resolve_internet_addr(network, address) + var socket = Socket(local_address=tcp_addr) + socket.bind(tcp_addr.ip, tcp_addr.port) + socket.set_socket_option(SO_REUSEADDR, 1) + socket.listen() + print(String("Listening on ") + socket.local_address) + return TCPListener(socket^, self, network, address) + + +trait Listener(Movable): + # Raising here because a Result[Optional[Connection], Error] is funky. + fn accept(self) raises -> Connection: + ... + + fn close(inout self) -> Error: + ... + + fn addr(self) raises -> TCPAddr: + ... + + +@value +struct TCPConnection(Conn): + """TCPConn is an implementation of the Conn interface for TCP network connections. + + Args: + connection: The underlying Connection. + """ + + var _connection: Connection + + fn __init__(inout self, connection: Connection): + self._connection = connection + + fn __init__(inout self, owned socket: Socket): + self._connection = Connection(socket^) + + fn __moveinit__(inout self, owned existing: Self): + self._connection = existing._connection^ + + fn read(inout self, inout dest: List[Byte]) -> (Int, Error): + """Reads data from the underlying file descriptor. + + Args: + dest: The buffer to read data into. + + Returns: + The number of bytes read, or an error if one occurred. + """ + var bytes_written: Int + var err: Error + bytes_written, err = self._connection.read(dest) + if err: + if str(err) != io.EOF: + return 0, err + + return bytes_written, Error() + + fn write(inout self, src: List[Byte]) -> (Int, Error): + """Writes data to the underlying file descriptor. + + Args: + src: The buffer to read data into. + + Returns: + The number of bytes written, or an error if one occurred. + """ + var bytes_written: Int + var err: Error + bytes_written, err = self._connection.write(src) + if err: + return 0, err + + return bytes_written, Error() + + fn close(inout self) -> Error: + """Closes the underlying file descriptor. + + Returns: + An error if one occurred, or None if the file descriptor was closed successfully. + """ + return self._connection.close() + + fn local_address(self) -> TCPAddr: + """Returns the local network address. + The Addr returned is shared by all invocations of local_address, so do not modify it. + + Returns: + The local network address. + """ + return self._connection.local_address() + + fn remote_address(self) -> TCPAddr: + """Returns the remote network address. + The Addr returned is shared by all invocations of remote_address, so do not modify it. + + Returns: + The remote network address. + """ + return self._connection.remote_address() + + +fn listen_tcp(network: String, local_address: TCPAddr) raises -> TCPListener: + """Creates a new TCP listener. + + Args: + network: The network type. + local_address: The local address to listen on. + """ + return ListenConfig(DEFAULT_TCP_KEEP_ALIVE).listen(network, local_address.ip + ":" + str(local_address.port)) + + +fn listen_tcp(network: String, local_address: String) raises -> TCPListener: + """Creates a new TCP listener. + + Args: + network: The network type. + local_address: The address to listen on. The format is "host:port". + """ + return ListenConfig(DEFAULT_TCP_KEEP_ALIVE).listen(network, local_address) + + +struct TCPListener(Listener): + var _file_descriptor: Socket + var listen_config: ListenConfig + var network_type: String + var address: String + + fn __init__( + inout self, + owned file_descriptor: Socket, + listen_config: ListenConfig, + network_type: String, + address: String, + ): + self._file_descriptor = file_descriptor^ + self.listen_config = listen_config + self.network_type = network_type + self.address = address + + fn __moveinit__(inout self, owned existing: Self): + self._file_descriptor = existing._file_descriptor^ + self.listen_config = existing.listen_config^ + self.network_type = existing.network_type + self.address = existing.address + + fn listen(self) raises -> Self: + return self.listen_config.listen(self.network_type, self.address) + + fn accept(self) raises -> Connection: + return Connection(self._file_descriptor.accept()) + + fn accept_tcp(self) raises -> TCPConnection: + return TCPConnection(self._file_descriptor.accept()) + + fn close(inout self) -> Error: + return self._file_descriptor.close() + + fn addr(self) raises -> TCPAddr: + return resolve_internet_addr(self.network_type, self.address) diff --git a/external/gojo/syscall/__init__.mojo b/external/gojo/syscall/__init__.mojo new file mode 100644 index 0000000..e69de29 diff --git a/external/gojo/syscall/file.mojo b/external/gojo/syscall/file.mojo new file mode 100644 index 0000000..d4095a5 --- /dev/null +++ b/external/gojo/syscall/file.mojo @@ -0,0 +1,110 @@ +from .types import c_int, c_char, c_void, c_size_t, c_ssize_t + + +# --- ( File Related Syscalls & Structs )--------------------------------------- +alias O_NONBLOCK = 16384 +alias O_ACCMODE = 3 +alias O_CLOEXEC = 524288 + + +fn close(fildes: c_int) -> c_int: + """Libc POSIX `close` function + Reference: https://man7.org/linux/man-pages/man3/close.3p.html + Fn signature: int close(int fildes). + + Args: + fildes: A File Descriptor to close. + + Returns: + Upon successful completion, 0 shall be returned; otherwise, -1 + shall be returned and errno set to indicate the error. + """ + return external_call["close", c_int, c_int](fildes) + + +fn open[*T: AnyType](path: Pointer[c_char], oflag: c_int, *args: *T) -> c_int: + """Libc POSIX `open` function + Reference: https://man7.org/linux/man-pages/man3/open.3p.html + Fn signature: int open(const char *path, int oflag, ...). + + Args: + path: A pointer to a C string containing the path to open. + oflag: The flags to open the file with. + args: The optional arguments. + Returns: + A File Descriptor or -1 in case of failure + """ + return external_call["open", c_int, Pointer[c_char], c_int](path, oflag, args) # FnName, RetType # Args + + +fn openat[*T: AnyType](fd: c_int, path: Pointer[c_char], oflag: c_int, *args: *T) -> c_int: + """Libc POSIX `open` function + Reference: https://man7.org/linux/man-pages/man3/open.3p.html + Fn signature: int openat(int fd, const char *path, int oflag, ...). + + Args: + fd: A File Descriptor. + path: A pointer to a C string containing the path to open. + oflag: The flags to open the file with. + args: The optional arguments. + Returns: + A File Descriptor or -1 in case of failure + """ + return external_call["openat", c_int, c_int, Pointer[c_char], c_int]( # FnName, RetType # Args + fd, path, oflag, args + ) + + +fn printf[*T: AnyType](format: Pointer[c_char], *args: *T) -> c_int: + """Libc POSIX `printf` function + Reference: https://man7.org/linux/man-pages/man3/fprintf.3p.html + Fn signature: int printf(const char *restrict format, ...). + + Args: format: A pointer to a C string containing the format. + args: The optional arguments. + Returns: The number of bytes written or -1 in case of failure. + """ + return external_call[ + "printf", + c_int, # FnName, RetType + Pointer[c_char], # Args + ](format, args) + + +fn sprintf[*T: AnyType](s: Pointer[c_char], format: Pointer[c_char], *args: *T) -> c_int: + """Libc POSIX `sprintf` function + Reference: https://man7.org/linux/man-pages/man3/fprintf.3p.html + Fn signature: int sprintf(char *restrict s, const char *restrict format, ...). + + Args: s: A pointer to a buffer to store the result. + format: A pointer to a C string containing the format. + args: The optional arguments. + Returns: The number of bytes written or -1 in case of failure. + """ + return external_call["sprintf", c_int, Pointer[c_char], Pointer[c_char]](s, format, args) # FnName, RetType # Args + + +fn read(fildes: c_int, buf: Pointer[c_void], nbyte: c_size_t) -> c_int: + """Libc POSIX `read` function + Reference: https://man7.org/linux/man-pages/man3/read.3p.html + Fn signature: sssize_t read(int fildes, void *buf, size_t nbyte). + + Args: fildes: A File Descriptor. + buf: A pointer to a buffer to store the read data. + nbyte: The number of bytes to read. + Returns: The number of bytes read or -1 in case of failure. + """ + return external_call["read", c_ssize_t, c_int, Pointer[c_void], c_size_t](fildes, buf, nbyte) + + +fn write(fildes: c_int, buf: Pointer[c_void], nbyte: c_size_t) -> c_int: + """Libc POSIX `write` function + Reference: https://man7.org/linux/man-pages/man3/write.3p.html + Fn signature: ssize_t write(int fildes, const void *buf, size_t nbyte). + + Args: fildes: A File Descriptor. + buf: A pointer to a buffer to write. + nbyte: The number of bytes to write. + Returns: The number of bytes written or -1 in case of failure. + """ + return external_call["write", c_ssize_t, c_int, Pointer[c_void], c_size_t](fildes, buf, nbyte) diff --git a/external/gojo/syscall/net.mojo b/external/gojo/syscall/net.mojo new file mode 100644 index 0000000..e396d3f --- /dev/null +++ b/external/gojo/syscall/net.mojo @@ -0,0 +1,749 @@ +from .types import c_char, c_int, c_ushort, c_uint, c_void, c_size_t, c_ssize_t, strlen +from .file import O_CLOEXEC, O_NONBLOCK + +alias IPPROTO_IPV6 = 41 +alias IPV6_V6ONLY = 26 +alias EPROTONOSUPPORT = 93 + +# Adapted from https://github.com/gabrieldemarmiesse/mojo-stdlib-extensions/ . Huge thanks to Gabriel! + +alias FD_STDIN: c_int = 0 +alias FD_STDOUT: c_int = 1 +alias FD_STDERR: c_int = 2 + +alias SUCCESS = 0 +alias GRND_NONBLOCK: UInt8 = 1 + +alias char_pointer = UnsafePointer[c_char] + + +# --- ( error.h Constants )----------------------------------------------------- +alias EPERM = 1 +alias ENOENT = 2 +alias ESRCH = 3 +alias EINTR = 4 +alias EIO = 5 +alias ENXIO = 6 +alias E2BIG = 7 +alias ENOEXEC = 8 +alias EBADF = 9 +alias ECHILD = 10 +alias EAGAIN = 11 +alias ENOMEM = 12 +alias EACCES = 13 +alias EFAULT = 14 +alias ENOTBLK = 15 +alias EBUSY = 16 +alias EEXIST = 17 +alias EXDEV = 18 +alias ENODEV = 19 +alias ENOTDIR = 20 +alias EISDIR = 21 +alias EINVAL = 22 +alias ENFILE = 23 +alias EMFILE = 24 +alias ENOTTY = 25 +alias ETXTBSY = 26 +alias EFBIG = 27 +alias ENOSPC = 28 +alias ESPIPE = 29 +alias EROFS = 30 +alias EMLINK = 31 +alias EPIPE = 32 +alias EDOM = 33 +alias ERANGE = 34 +alias EWOULDBLOCK = EAGAIN + + +fn to_char_ptr(s: String) -> Pointer[c_char]: + """Only ASCII-based strings.""" + var ptr = Pointer[c_char]().alloc(len(s)) + for i in range(len(s)): + ptr.store(i, ord(s[i])) + return ptr + + +fn c_charptr_to_string(s: Pointer[c_char]) -> String: + return String(s.bitcast[Int8](), strlen(s)) + + +fn cftob(val: c_int) -> Bool: + """Convert C-like failure (-1) to Bool.""" + return rebind[Bool](val > 0) + + +# --- ( Network Related Constants )--------------------------------------------- +alias sa_family_t = c_ushort +alias socklen_t = c_uint +alias in_addr_t = c_uint +alias in_port_t = c_ushort + +# Address Family Constants +alias AF_UNSPEC = 0 +alias AF_UNIX = 1 +alias AF_LOCAL = AF_UNIX +alias AF_INET = 2 +alias AF_AX25 = 3 +alias AF_IPX = 4 +alias AF_APPLETALK = 5 +alias AF_NETROM = 6 +alias AF_BRIDGE = 7 +alias AF_ATMPVC = 8 +alias AF_X25 = 9 +alias AF_INET6 = 10 +alias AF_ROSE = 11 +alias AF_DECnet = 12 +alias AF_NETBEUI = 13 +alias AF_SECURITY = 14 +alias AF_KEY = 15 +alias AF_NETLINK = 16 +alias AF_ROUTE = AF_NETLINK +alias AF_PACKET = 17 +alias AF_ASH = 18 +alias AF_ECONET = 19 +alias AF_ATMSVC = 20 +alias AF_RDS = 21 +alias AF_SNA = 22 +alias AF_IRDA = 23 +alias AF_PPPOX = 24 +alias AF_WANPIPE = 25 +alias AF_LLC = 26 +alias AF_CAN = 29 +alias AF_TIPC = 30 +alias AF_BLUETOOTH = 31 +alias AF_IUCV = 32 +alias AF_RXRPC = 33 +alias AF_ISDN = 34 +alias AF_PHONET = 35 +alias AF_IEEE802154 = 36 +alias AF_CAIF = 37 +alias AF_ALG = 38 +alias AF_NFC = 39 +alias AF_VSOCK = 40 +alias AF_KCM = 41 +alias AF_QIPCRTR = 42 +alias AF_MAX = 43 + +# Protocol family constants +alias PF_UNSPEC = AF_UNSPEC +alias PF_UNIX = AF_UNIX +alias PF_LOCAL = AF_LOCAL +alias PF_INET = AF_INET +alias PF_AX25 = AF_AX25 +alias PF_IPX = AF_IPX +alias PF_APPLETALK = AF_APPLETALK +alias PF_NETROM = AF_NETROM +alias PF_BRIDGE = AF_BRIDGE +alias PF_ATMPVC = AF_ATMPVC +alias PF_X25 = AF_X25 +alias PF_INET6 = AF_INET6 +alias PF_ROSE = AF_ROSE +alias PF_DECnet = AF_DECnet +alias PF_NETBEUI = AF_NETBEUI +alias PF_SECURITY = AF_SECURITY +alias PF_KEY = AF_KEY +alias PF_NETLINK = AF_NETLINK +alias PF_ROUTE = AF_ROUTE +alias PF_PACKET = AF_PACKET +alias PF_ASH = AF_ASH +alias PF_ECONET = AF_ECONET +alias PF_ATMSVC = AF_ATMSVC +alias PF_RDS = AF_RDS +alias PF_SNA = AF_SNA +alias PF_IRDA = AF_IRDA +alias PF_PPPOX = AF_PPPOX +alias PF_WANPIPE = AF_WANPIPE +alias PF_LLC = AF_LLC +alias PF_CAN = AF_CAN +alias PF_TIPC = AF_TIPC +alias PF_BLUETOOTH = AF_BLUETOOTH +alias PF_IUCV = AF_IUCV +alias PF_RXRPC = AF_RXRPC +alias PF_ISDN = AF_ISDN +alias PF_PHONET = AF_PHONET +alias PF_IEEE802154 = AF_IEEE802154 +alias PF_CAIF = AF_CAIF +alias PF_ALG = AF_ALG +alias PF_NFC = AF_NFC +alias PF_VSOCK = AF_VSOCK +alias PF_KCM = AF_KCM +alias PF_QIPCRTR = AF_QIPCRTR +alias PF_MAX = AF_MAX + +# Socket Type constants +alias SOCK_STREAM = 1 +alias SOCK_DGRAM = 2 +alias SOCK_RAW = 3 +alias SOCK_RDM = 4 +alias SOCK_SEQPACKET = 5 +alias SOCK_DCCP = 6 +alias SOCK_PACKET = 10 +alias SOCK_CLOEXEC = O_CLOEXEC +alias SOCK_NONBLOCK = O_NONBLOCK + +# Address Information +alias AI_PASSIVE = 1 +alias AI_CANONNAME = 2 +alias AI_NUMERICHOST = 4 +alias AI_V4MAPPED = 2048 +alias AI_ALL = 256 +alias AI_ADDRCONFIG = 1024 +alias AI_IDN = 64 + +alias INET_ADDRSTRLEN = 16 +alias INET6_ADDRSTRLEN = 46 + +alias SHUT_RD = 0 +alias SHUT_WR = 1 +alias SHUT_RDWR = 2 + +alias SOL_SOCKET = 65535 + +# Socket Options +alias SO_DEBUG = 1 +alias SO_REUSEADDR = 4 +alias SO_TYPE = 4104 +alias SO_ERROR = 4103 +alias SO_DONTROUTE = 16 +alias SO_BROADCAST = 32 +alias SO_SNDBUF = 4097 +alias SO_RCVBUF = 4098 +alias SO_KEEPALIVE = 8 +alias SO_OOBINLINE = 256 +alias SO_LINGER = 128 +alias SO_REUSEPORT = 512 +alias SO_RCVLOWAT = 4100 +alias SO_SNDLOWAT = 4099 +alias SO_RCVTIMEO = 4102 +alias SO_SNDTIMEO = 4101 +alias SO_RCVTIMEO_OLD = 4102 +alias SO_SNDTIMEO_OLD = 4101 +alias SO_ACCEPTCONN = 2 + +# unsure of these socket options, they weren't available via python +alias SO_NO_CHECK = 11 +alias SO_PRIORITY = 12 +alias SO_BSDCOMPAT = 14 +alias SO_PASSCRED = 16 +alias SO_PEERCRED = 17 +alias SO_SECURITY_AUTHENTICATION = 22 +alias SO_SECURITY_ENCRYPTION_TRANSPORT = 23 +alias SO_SECURITY_ENCRYPTION_NETWORK = 24 +alias SO_BINDTODEVICE = 25 +alias SO_ATTACH_FILTER = 26 +alias SO_DETACH_FILTER = 27 +alias SO_GET_FILTER = SO_ATTACH_FILTER +alias SO_PEERNAME = 28 +alias SO_TIMESTAMP = 29 +alias SO_TIMESTAMP_OLD = 29 +alias SO_PEERSEC = 31 +alias SO_SNDBUFFORCE = 32 +alias SO_RCVBUFFORCE = 33 +alias SO_PASSSEC = 34 +alias SO_TIMESTAMPNS = 35 +alias SO_TIMESTAMPNS_OLD = 35 +alias SO_MARK = 36 +alias SO_TIMESTAMPING = 37 +alias SO_TIMESTAMPING_OLD = 37 +alias SO_PROTOCOL = 38 +alias SO_DOMAIN = 39 +alias SO_RXQ_OVFL = 40 +alias SO_WIFI_STATUS = 41 +alias SCM_WIFI_STATUS = SO_WIFI_STATUS +alias SO_PEEK_OFF = 42 +alias SO_NOFCS = 43 +alias SO_LOCK_FILTER = 44 +alias SO_SELECT_ERR_QUEUE = 45 +alias SO_BUSY_POLL = 46 +alias SO_MAX_PACING_RATE = 47 +alias SO_BPF_EXTENSIONS = 48 +alias SO_INCOMING_CPU = 49 +alias SO_ATTACH_BPF = 50 +alias SO_DETACH_BPF = SO_DETACH_FILTER +alias SO_ATTACH_REUSEPORT_CBPF = 51 +alias SO_ATTACH_REUSEPORT_EBPF = 52 +alias SO_CNX_ADVICE = 53 +alias SCM_TIMESTAMPING_OPT_STATS = 54 +alias SO_MEMINFO = 55 +alias SO_INCOMING_NAPI_ID = 56 +alias SO_COOKIE = 57 +alias SCM_TIMESTAMPING_PKTINFO = 58 +alias SO_PEERGROUPS = 59 +alias SO_ZEROCOPY = 60 +alias SO_TXTIME = 61 +alias SCM_TXTIME = SO_TXTIME +alias SO_BINDTOIFINDEX = 62 +alias SO_TIMESTAMP_NEW = 63 +alias SO_TIMESTAMPNS_NEW = 64 +alias SO_TIMESTAMPING_NEW = 65 +alias SO_RCVTIMEO_NEW = 66 +alias SO_SNDTIMEO_NEW = 67 +alias SO_DETACH_REUSEPORT_BPF = 68 + + +# --- ( Network Related Structs )----------------------------------------------- +@value +@register_passable("trivial") +struct in_addr: + var s_addr: in_addr_t + + +@value +@register_passable("trivial") +struct in6_addr: + var s6_addr: StaticTuple[c_char, 16] + + +@value +@register_passable("trivial") +struct sockaddr: + var sa_family: sa_family_t + var sa_data: StaticTuple[c_char, 14] + + +@value +@register_passable("trivial") +struct sockaddr_in: + var sin_family: sa_family_t + var sin_port: in_port_t + var sin_addr: in_addr + var sin_zero: StaticTuple[c_char, 8] + + +@value +@register_passable("trivial") +struct sockaddr_in6: + var sin6_family: sa_family_t + var sin6_port: in_port_t + var sin6_flowinfo: c_uint + var sin6_addr: in6_addr + var sin6_scope_id: c_uint + + +@value +@register_passable("trivial") +struct addrinfo: + """Struct field ordering can vary based on platform. + For MacOS, I had to swap the order of ai_canonname and ai_addr. + https://stackoverflow.com/questions/53575101/calling-getaddrinfo-directly-from-python-ai-addr-is-null-pointer. + """ + + var ai_flags: c_int + var ai_family: c_int + var ai_socktype: c_int + var ai_protocol: c_int + var ai_addrlen: socklen_t + var ai_canonname: Pointer[c_char] + var ai_addr: Pointer[sockaddr] + var ai_next: Pointer[addrinfo] + + fn __init__() -> Self: + return Self(0, 0, 0, 0, 0, Pointer[c_char](), Pointer[sockaddr](), Pointer[addrinfo]()) + + +@value +@register_passable("trivial") +struct addrinfo_unix: + """Struct field ordering can vary based on platform. + For MacOS, I had to swap the order of ai_canonname and ai_addr. + https://stackoverflow.com/questions/53575101/calling-getaddrinfo-directly-from-python-ai-addr-is-null-pointer. + """ + + var ai_flags: c_int + var ai_family: c_int + var ai_socktype: c_int + var ai_protocol: c_int + var ai_addrlen: socklen_t + var ai_addr: Pointer[sockaddr] + var ai_canonname: Pointer[c_char] + var ai_next: Pointer[addrinfo] + + fn __init__() -> Self: + return Self(0, 0, 0, 0, 0, Pointer[sockaddr](), Pointer[c_char](), Pointer[addrinfo]()) + + +# --- ( Network Related Syscalls & Structs )------------------------------------ + + +fn htonl(hostlong: c_uint) -> c_uint: + """Libc POSIX `htonl` function + Reference: https://man7.org/linux/man-pages/man3/htonl.3p.html + Fn signature: uint32_t htonl(uint32_t hostlong). + + Args: hostlong: A 32-bit integer in host byte order. + Returns: The value provided in network byte order. + """ + return external_call["htonl", c_uint, c_uint](hostlong) + + +fn htons(hostshort: c_ushort) -> c_ushort: + """Libc POSIX `htons` function + Reference: https://man7.org/linux/man-pages/man3/htonl.3p.html + Fn signature: uint16_t htons(uint16_t hostshort). + + Args: hostshort: A 16-bit integer in host byte order. + Returns: The value provided in network byte order. + """ + return external_call["htons", c_ushort, c_ushort](hostshort) + + +fn ntohl(netlong: c_uint) -> c_uint: + """Libc POSIX `ntohl` function + Reference: https://man7.org/linux/man-pages/man3/htonl.3p.html + Fn signature: uint32_t ntohl(uint32_t netlong). + + Args: netlong: A 32-bit integer in network byte order. + Returns: The value provided in host byte order. + """ + return external_call["ntohl", c_uint, c_uint](netlong) + + +fn ntohs(netshort: c_ushort) -> c_ushort: + """Libc POSIX `ntohs` function + Reference: https://man7.org/linux/man-pages/man3/htonl.3p.html + Fn signature: uint16_t ntohs(uint16_t netshort). + + Args: netshort: A 16-bit integer in network byte order. + Returns: The value provided in host byte order. + """ + return external_call["ntohs", c_ushort, c_ushort](netshort) + + +fn inet_ntop(af: c_int, src: Pointer[c_void], dst: Pointer[c_char], size: socklen_t) -> Pointer[c_char]: + """Libc POSIX `inet_ntop` function + Reference: https://man7.org/linux/man-pages/man3/inet_ntop.3p.html. + Fn signature: const char *inet_ntop(int af, const void *restrict src, char *restrict dst, socklen_t size). + + Args: + af: Address Family see AF_ aliases. + src: A pointer to a binary address. + dst: A pointer to a buffer to store the result. + size: The size of the buffer. + + Returns: + A pointer to the buffer containing the result. + """ + return external_call[ + "inet_ntop", + Pointer[c_char], # FnName, RetType + c_int, + Pointer[c_void], + Pointer[c_char], + socklen_t, # Args + ](af, src, dst, size) + + +fn inet_pton(af: c_int, src: Pointer[c_char], dst: Pointer[c_void]) -> c_int: + """Libc POSIX `inet_pton` function + Reference: https://man7.org/linux/man-pages/man3/inet_ntop.3p.html + Fn signature: int inet_pton(int af, const char *restrict src, void *restrict dst). + + Args: af: Address Family see AF_ aliases. + src: A pointer to a string containing the address. + dst: A pointer to a buffer to store the result. + Returns: 1 on success, 0 if the input is not a valid address, -1 on error. + """ + return external_call[ + "inet_pton", + c_int, # FnName, RetType + c_int, + Pointer[c_char], + Pointer[c_void], # Args + ](af, src, dst) + + +fn inet_addr(cp: Pointer[c_char]) -> in_addr_t: + """Libc POSIX `inet_addr` function + Reference: https://man7.org/linux/man-pages/man3/inet_addr.3p.html + Fn signature: in_addr_t inet_addr(const char *cp). + + Args: cp: A pointer to a string containing the address. + Returns: The address in network byte order. + """ + return external_call["inet_addr", in_addr_t, Pointer[c_char]](cp) + + +fn inet_ntoa(addr: in_addr) -> Pointer[c_char]: + """Libc POSIX `inet_ntoa` function + Reference: https://man7.org/linux/man-pages/man3/inet_addr.3p.html + Fn signature: char *inet_ntoa(struct in_addr in). + + Args: in: A pointer to a string containing the address. + Returns: The address in network byte order. + """ + return external_call["inet_ntoa", Pointer[c_char], in_addr](addr) + + +fn socket(domain: c_int, type: c_int, protocol: c_int) -> c_int: + """Libc POSIX `socket` function + Reference: https://man7.org/linux/man-pages/man3/socket.3p.html + Fn signature: int socket(int domain, int type, int protocol). + + Args: domain: Address Family see AF_ aliases. + type: Socket Type see SOCK_ aliases. + protocol: The protocol to use. + Returns: A File Descriptor or -1 in case of failure. + """ + return external_call["socket", c_int, c_int, c_int, c_int](domain, type, protocol) # FnName, RetType # Args + + +fn setsockopt( + socket: c_int, + level: c_int, + option_name: c_int, + option_value: Pointer[c_void], + option_len: socklen_t, +) -> c_int: + """Libc POSIX `setsockopt` function + Reference: https://man7.org/linux/man-pages/man3/setsockopt.3p.html + Fn signature: int setsockopt(int socket, int level, int option_name, const void *option_value, socklen_t option_len). + + Args: + socket: A File Descriptor. + level: The protocol level. + option_name: The option to set. + option_value: A pointer to the value to set. + option_len: The size of the value. + Returns: 0 on success, -1 on error. + """ + return external_call[ + "setsockopt", + c_int, # FnName, RetType + c_int, + c_int, + c_int, + Pointer[c_void], + socklen_t, # Args + ](socket, level, option_name, option_value, option_len) + + +fn getsockopt( + socket: c_int, + level: c_int, + option_name: c_int, + option_value: Pointer[c_void], + option_len: Pointer[socklen_t], +) -> c_int: + """Libc POSIX `getsockopt` function + Reference: https://man7.org/linux/man-pages/man3/getsockopt.3p.html + Fn signature: int getsockopt(int socket, int level, int option_name, void *restrict option_value, socklen_t *restrict option_len). + + Args: socket: A File Descriptor. + level: The protocol level. + option_name: The option to get. + option_value: A pointer to the value to get. + option_len: Pointer to the size of the value. + Returns: 0 on success, -1 on error. + """ + return external_call[ + "getsockopt", + c_int, # FnName, RetType + c_int, + c_int, + c_int, + Pointer[c_void], + Pointer[socklen_t], # Args + ](socket, level, option_name, option_value, option_len) + + +fn getsockname(socket: c_int, address: Pointer[sockaddr], address_len: Pointer[socklen_t]) -> c_int: + """Libc POSIX `getsockname` function + Reference: https://man7.org/linux/man-pages/man3/getsockname.3p.html + Fn signature: int getsockname(int socket, struct sockaddr *restrict address, socklen_t *restrict address_len). + + Args: socket: A File Descriptor. + address: A pointer to a buffer to store the address of the peer. + address_len: A pointer to the size of the buffer. + Returns: 0 on success, -1 on error. + """ + return external_call[ + "getsockname", + c_int, # FnName, RetType + c_int, + Pointer[sockaddr], + Pointer[socklen_t], # Args + ](socket, address, address_len) + + +fn getpeername(sockfd: c_int, addr: Pointer[sockaddr], address_len: Pointer[socklen_t]) -> c_int: + """Libc POSIX `getpeername` function + Reference: https://man7.org/linux/man-pages/man2/getpeername.2.html + Fn signature: int getpeername(int socket, struct sockaddr *restrict addr, socklen_t *restrict address_len). + + Args: sockfd: A File Descriptor. + addr: A pointer to a buffer to store the address of the peer. + address_len: A pointer to the size of the buffer. + Returns: 0 on success, -1 on error. + """ + return external_call[ + "getpeername", + c_int, # FnName, RetType + c_int, + Pointer[sockaddr], + Pointer[socklen_t], # Args + ](sockfd, addr, address_len) + + +fn bind(socket: c_int, address: Pointer[sockaddr], address_len: socklen_t) -> c_int: + """Libc POSIX `bind` function + Reference: https://man7.org/linux/man-pages/man3/bind.3p.html + Fn signature: int bind(int socket, const struct sockaddr *address, socklen_t address_len). + """ + return external_call["bind", c_int, c_int, Pointer[sockaddr], socklen_t]( # FnName, RetType # Args + socket, address, address_len + ) + + +fn listen(socket: c_int, backlog: c_int) -> c_int: + """Libc POSIX `listen` function + Reference: https://man7.org/linux/man-pages/man3/listen.3p.html + Fn signature: int listen(int socket, int backlog). + + Args: socket: A File Descriptor. + backlog: The maximum length of the queue of pending connections. + Returns: 0 on success, -1 on error. + """ + return external_call["listen", c_int, c_int, c_int](socket, backlog) + + +fn accept(socket: c_int, address: Pointer[sockaddr], address_len: Pointer[socklen_t]) -> c_int: + """Libc POSIX `accept` function + Reference: https://man7.org/linux/man-pages/man3/accept.3p.html + Fn signature: int accept(int socket, struct sockaddr *restrict address, socklen_t *restrict address_len). + + Args: socket: A File Descriptor. + address: A pointer to a buffer to store the address of the peer. + address_len: A pointer to the size of the buffer. + Returns: A File Descriptor or -1 in case of failure. + """ + return external_call[ + "accept", + c_int, # FnName, RetType + c_int, + Pointer[sockaddr], + Pointer[socklen_t], # Args + ](socket, address, address_len) + + +fn connect(socket: c_int, address: Pointer[sockaddr], address_len: socklen_t) -> c_int: + """Libc POSIX `connect` function + Reference: https://man7.org/linux/man-pages/man3/connect.3p.html + Fn signature: int connect(int socket, const struct sockaddr *address, socklen_t address_len). + + Args: socket: A File Descriptor. + address: A pointer to the address to connect to. + address_len: The size of the address. + Returns: 0 on success, -1 on error. + """ + return external_call["connect", c_int, c_int, Pointer[sockaddr], socklen_t]( # FnName, RetType # Args + socket, address, address_len + ) + + +fn recv(socket: c_int, buffer: Pointer[c_void], length: c_size_t, flags: c_int) -> c_ssize_t: + """Libc POSIX `recv` function + Reference: https://man7.org/linux/man-pages/man3/recv.3p.html + Fn signature: ssize_t recv(int socket, void *buffer, size_t length, int flags). + """ + return external_call[ + "recv", + c_ssize_t, # FnName, RetType + c_int, + Pointer[c_void], + c_size_t, + c_int, # Args + ](socket, buffer, length, flags) + + +fn send(socket: c_int, buffer: Pointer[c_void], length: c_size_t, flags: c_int) -> c_ssize_t: + """Libc POSIX `send` function + Reference: https://man7.org/linux/man-pages/man3/send.3p.html + Fn signature: ssize_t send(int socket, const void *buffer, size_t length, int flags). + + Args: socket: A File Descriptor. + buffer: A pointer to the buffer to send. + length: The size of the buffer. + flags: Flags to control the behaviour of the function. + Returns: The number of bytes sent or -1 in case of failure. + """ + return external_call[ + "send", + c_ssize_t, # FnName, RetType + c_int, + Pointer[c_void], + c_size_t, + c_int, # Args + ](socket, buffer, length, flags) + + +fn shutdown(socket: c_int, how: c_int) -> c_int: + """Libc POSIX `shutdown` function + Reference: https://man7.org/linux/man-pages/man3/shutdown.3p.html + Fn signature: int shutdown(int socket, int how). + + Args: socket: A File Descriptor. + how: How to shutdown the socket. + Returns: 0 on success, -1 on error. + """ + return external_call["shutdown", c_int, c_int, c_int](socket, how) # FnName, RetType # Args + + +fn getaddrinfo( + nodename: Pointer[c_char], + servname: Pointer[c_char], + hints: Pointer[addrinfo], + res: Pointer[Pointer[addrinfo]], +) -> c_int: + """Libc POSIX `getaddrinfo` function + Reference: https://man7.org/linux/man-pages/man3/getaddrinfo.3p.html + Fn signature: int getaddrinfo(const char *restrict nodename, const char *restrict servname, const struct addrinfo *restrict hints, struct addrinfo **restrict res). + """ + return external_call[ + "getaddrinfo", + c_int, # FnName, RetType + Pointer[c_char], + Pointer[c_char], + Pointer[addrinfo], # Args + Pointer[Pointer[addrinfo]], # Args + ](nodename, servname, hints, res) + + +fn getaddrinfo_unix( + nodename: Pointer[c_char], + servname: Pointer[c_char], + hints: Pointer[addrinfo_unix], + res: Pointer[Pointer[addrinfo_unix]], +) -> c_int: + """Libc POSIX `getaddrinfo` function + Reference: https://man7.org/linux/man-pages/man3/getaddrinfo.3p.html + Fn signature: int getaddrinfo(const char *restrict nodename, const char *restrict servname, const struct addrinfo *restrict hints, struct addrinfo **restrict res). + """ + return external_call[ + "getaddrinfo", + c_int, # FnName, RetType + Pointer[c_char], + Pointer[c_char], + Pointer[addrinfo_unix], # Args + Pointer[Pointer[addrinfo_unix]], # Args + ](nodename, servname, hints, res) + + +fn gai_strerror(ecode: c_int) -> Pointer[c_char]: + """Libc POSIX `gai_strerror` function + Reference: https://man7.org/linux/man-pages/man3/gai_strerror.3p.html + Fn signature: const char *gai_strerror(int ecode). + + Args: ecode: The error code. + Returns: A pointer to a string describing the error. + """ + return external_call["gai_strerror", Pointer[c_char], c_int](ecode) # FnName, RetType # Args + + +fn inet_pton(address_family: Int, address: String) -> Int: + var ip_buf_size = 4 + if address_family == AF_INET6: + ip_buf_size = 16 + + var ip_buf = Pointer[c_void].alloc(ip_buf_size) + var conv_status = inet_pton(rebind[c_int](address_family), to_char_ptr(address), ip_buf) + return int(ip_buf.bitcast[c_uint]().load()) diff --git a/external/gojo/syscall/types.mojo b/external/gojo/syscall/types.mojo new file mode 100644 index 0000000..56693e7 --- /dev/null +++ b/external/gojo/syscall/types.mojo @@ -0,0 +1,63 @@ +@value +struct Str: + var vector: List[c_char] + + fn __init__(inout self, string: String): + self.vector = List[c_char](capacity=len(string) + 1) + for i in range(len(string)): + self.vector.append(ord(string[i])) + self.vector.append(0) + + fn __init__(inout self, size: Int): + self.vector = List[c_char]() + self.vector.resize(size + 1, 0) + + fn __len__(self) -> Int: + for i in range(len(self.vector)): + if self.vector[i] == 0: + return i + return -1 + + fn to_string(self, size: Int) -> String: + var result: String = "" + for i in range(size): + result += chr(int(self.vector[i])) + return result + + fn __enter__(owned self: Self) -> Self: + return self^ + + +fn strlen(s: Pointer[c_char]) -> c_size_t: + """Libc POSIX `strlen` function + Reference: https://man7.org/linux/man-pages/man3/strlen.3p.html + Fn signature: size_t strlen(const char *s). + + Args: s: A pointer to a C string. + Returns: The length of the string. + """ + return external_call["strlen", c_size_t, Pointer[c_char]](s) + + +# Adapted from https://github.com/crisadamo/mojo-Libc . Huge thanks to Cristian! +# C types +alias c_void = UInt8 +alias c_char = UInt8 +alias c_schar = Int8 +alias c_uchar = UInt8 +alias c_short = Int16 +alias c_ushort = UInt16 +alias c_int = Int32 +alias c_uint = UInt32 +alias c_long = Int64 +alias c_ulong = UInt64 +alias c_float = Float32 +alias c_double = Float64 + +# `Int` is known to be machine's width +alias c_size_t = Int +alias c_ssize_t = Int + +alias ptrdiff_t = Int64 +alias intptr_t = Int64 +alias uintptr_t = UInt64 diff --git a/external/gojo/unicode/__init__.mojo b/external/gojo/unicode/__init__.mojo new file mode 100644 index 0000000..bd4cba6 --- /dev/null +++ b/external/gojo/unicode/__init__.mojo @@ -0,0 +1 @@ +from .utf8 import string_iterator, rune_count_in_string diff --git a/external/gojo/unicode/utf8/__init__.mojo b/external/gojo/unicode/utf8/__init__.mojo new file mode 100644 index 0000000..b8732ec --- /dev/null +++ b/external/gojo/unicode/utf8/__init__.mojo @@ -0,0 +1,4 @@ +"""Almost all of the actual implementation in this module was written by @mzaks (https://github.com/mzaks)! +This would not be possible without his help. +""" +from .runes import string_iterator, rune_count_in_string diff --git a/external/gojo/unicode/utf8/runes.mojo b/external/gojo/unicode/utf8/runes.mojo new file mode 100644 index 0000000..56da84b --- /dev/null +++ b/external/gojo/unicode/utf8/runes.mojo @@ -0,0 +1,353 @@ +"""Almost all of the actual implementation in this module was written by @mzaks (https://github.com/mzaks)! +This would not be possible without his help. +""" + +from ...builtins import Rune +from algorithm.functional import vectorize +from memory.unsafe import DTypePointer +from sys.info import simdwidthof +from math.bit import ctlz + + +# The default lowest and highest continuation byte. +alias locb = 0b10000000 +alias hicb = 0b10111111 +alias RUNE_SELF = 0x80 # Characters below RuneSelf are represented as themselves in a single byte + + +# acceptRange gives the range of valid values for the second byte in a UTF-8 +# sequence. +@value +struct AcceptRange(CollectionElement): + var lo: UInt8 # lowest value for second byte. + var hi: UInt8 # highest value for second byte. + + +# ACCEPT_RANGES has size 16 to avoid bounds checks in the code that uses it. +alias ACCEPT_RANGES = List[AcceptRange]( + AcceptRange(locb, hicb), + AcceptRange(0xA0, hicb), + AcceptRange(locb, 0x9F), + AcceptRange(0x90, hicb), + AcceptRange(locb, 0x8F), +) + +# These names of these constants are chosen to give nice alignment in the +# table below. The first nibble is an index into acceptRanges or F for +# special one-byte cases. The second nibble is the Rune length or the +# Status for the special one-byte case. +alias xx = 0xF1 # invalid: size 1 +alias as1 = 0xF0 # ASCII: size 1 +alias s1 = 0x02 # accept 0, size 2 +alias s2 = 0x13 # accept 1, size 3 +alias s3 = 0x03 # accept 0, size 3 +alias s4 = 0x23 # accept 2, size 3 +alias s5 = 0x34 # accept 3, size 4 +alias s6 = 0x04 # accept 0, size 4 +alias s7 = 0x44 # accept 4, size 4 + + +# first is information about the first byte in a UTF-8 sequence. +var first = List[UInt8]( + # 1 2 3 4 5 6 7 8 9 A B C D E F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x00-0x0F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x10-0x1F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x20-0x2F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x30-0x3F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x40-0x4F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x50-0x5F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x60-0x6F + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, + as1, # 0x70-0x7F + # 1 2 3 4 5 6 7 8 9 A B C D E F + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, # 0x80-0x8F + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, # 0x90-0x9F + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, # 0xA0-0xAF + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, # 0xB0-0xBF + xx, + xx, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, # 0xC0-0xCF + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, + s1, # 0xD0-0xDF + s2, + s3, + s3, + s3, + s3, + s3, + s3, + s3, + s3, + s3, + s3, + s3, + s3, + s4, + s3, + s3, # 0xE0-0xEF + s5, + s6, + s6, + s6, + s7, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, + xx, # 0xF0-0xFF +) + + +alias simd_width_u8 = simdwidthof[DType.uint8]() + + +fn rune_count_in_string(s: String) -> Int: + """Count the number of runes in a string. + + Args: + s: The string to count runes in. + + Returns: + The number of runes in the string. + """ + var p = s._as_ptr().bitcast[DType.uint8]() + var string_byte_length = len(s) + var result = 0 + + @parameter + fn count[simd_width: Int](offset: Int): + result += int(((p.load[width=simd_width](offset) >> 6) != 0b10).cast[DType.uint8]().reduce_add()) + + vectorize[count, simd_width_u8](string_byte_length) + return result + + +fn string_iterator(s: String, func: fn (String) -> None): + """Iterate over the runes in a string and call the given function with each rune. + + Args: + s: The string to iterate over. + func: The function to call with each rune. + """ + var bytes = len(s) + var p = s._as_ptr().bitcast[DType.uint8]() + while bytes > 0: + var char_length = int((p.load() >> 7 == 0).cast[DType.uint8]() * 1 + ctlz(~p.load())) + var sp = DTypePointer[DType.int8].alloc(char_length + 1) + memcpy(sp, p.bitcast[DType.int8](), char_length) + sp[char_length] = 0 + func(String(sp, char_length + 1)) + bytes -= char_length + p += char_length diff --git a/external/hue/happy_palettegen.mojo b/external/hue/happy_palettegen.mojo index 0f05362..4678341 100644 --- a/external/hue/happy_palettegen.mojo +++ b/external/hue/happy_palettegen.mojo @@ -1,6 +1,6 @@ from random import randn_float64 from .color import Color, hsv, lab_to_hcl -from .soft_palettegen import soft_palette_ex, soft_palette_settings +from .soft_palettegen import soft_palette_ex, SoftPaletteSettings fn fast_happy_palette(colors_count: Int) -> List[Color]: diff --git a/external/mist/__init__.mojo b/external/mist/__init__.mojo index 45c8a02..459e17d 100644 --- a/external/mist/__init__.mojo +++ b/external/mist/__init__.mojo @@ -1,6 +1,6 @@ from .color import Color from .style import TerminalStyle -from .profile import Profile, ASCII, ANSI, ANSI256, TRUE_COLOR, AnyColor +from .profile import Profile, ASCII, ANSI, ANSI256, TRUE_COLOR, AnyColor, NoColor from .renderers import ( as_color, with_background_color, diff --git a/external/mist/color.mojo b/external/mist/color.mojo index ccc8ba2..fa455f7 100644 --- a/external/mist/color.mojo +++ b/external/mist/color.mojo @@ -195,7 +195,7 @@ fn hex_to_rgb(value: String) -> hue.Color: value: Hex color value. Returns: - hue.Color color. + Color. """ var hex = value[1:] var indices = List[Int](0, 2, 4) @@ -278,7 +278,7 @@ fn hex_to_ansi256(color: hue.Color) -> ANSI256Color: """Converts a hex code to a ANSI256 color. Args: - color: hue.Color hex code. + color: Hex code color from hue.Color. """ # Calculate the nearest 0-based color index at 16..231 # Originally had * 255 in each of these diff --git a/external/mist/profile.mojo b/external/mist/profile.mojo index 77d1969..97069a5 100644 --- a/external/mist/profile.mojo +++ b/external/mist/profile.mojo @@ -40,12 +40,12 @@ fn get_color_profile() -> Profile: # COLORTERM is used by some terminals to indicate TRUE_COLOR support. if color_term == "24bit": pass - elif color_term == TRUE_COLOR: + elif color_term == "truecolor": if term.startswith("screen"): # tmux supports TRUE_COLOR, screen only ANSI256 if os.getenv("TERM_PROGRAM") != "tmux": return Profile(ANSI256) - return Profile(TRUE_COLOR) + return Profile(TRUE_COLOR) elif color_term == "yes": pass elif color_term == "true": @@ -63,7 +63,7 @@ fn get_color_profile() -> Profile: if "color" in term: return Profile(ANSI) - if ANSI in term: + if "ansi" in term: return Profile(ANSI) return Profile(ASCII)