Add async serialize option via coroutines

LibSerialize.lua

@@ -120,23 +120,77 @@ end
 
     Calls `SerializeEx(opts, ...)` with the default options (see below)
 
+* **`LibSerialize:SerializeAsyncEx(opts, ...)`**
+
+    Arguments:
+    * `opts`: options (see below)
+    * `...`: a variable number of serializable values
+
+    Returns:
+    * `handler`: function to run the process. This should be run until the
+      first returned value is false.
+      `handler` returns:
+      * `ongoing`: Boolean if there is more to process.
+      * `result`: `...` serialized as a string
+
+* **`LibSerialize:SerializeAsync(...)`**
+
+    Arguments:
+    * `...`: a variable number of serializable values
+
+    Returns:
+    * `handler`: function to run the process. This should be run until the
+      first returned value is false.
+      `handler` returns:
+      * `ongoing`: Boolean if there is more to process.
+      * `result`: `...` serialized as a string
+
+    Calls `SerializeAsyncEx(opts, ...)` with the default options (see below)
+
 * **`LibSerialize:Deserialize(input)`**
 
     Arguments:
     * `input`: a string previously returned from `LibSerialize:Serialize()`
+    * `opts`: options (see below)
 
     Returns:
     * `success`: a boolean indicating if deserialization was successful
     * `...`: the deserialized value(s), or a string containing the encountered Lua error
 
-* **`LibSerialize:DeserializeValue(input)`**
+* **`LibSerialize:DeserializeValue(input, opts)`**
 
     Arguments:
     * `input`: a string previously returned from `LibSerialize:Serialize()`
+    * `opts`: options (see below)
 
     Returns:
     * `...`: the deserialized value(s)
 
+* **`LibSerialize:DeserializeAsync(input)`**
+
+    Arguments:
+    * `input`: a string previously returned from `LibSerialize:Serialize()`
+
+    Returns:
+    * `handler`: function to run the process. This should be run until the
+      first returned value is false. The remaining return values match `Deserialize()`.
+      `handler` returns:
+      * `success`: a boolean indicating if deserialization was successful
+      * `...`: the deserialized value(s), or a string containing the encountered Lua error
+
+* **`LibSerialize:DeserializeAsyncValue(input, opts)`**
+
+    Arguments:
+    * `input`: a string previously returned from `LibSerialize:Serialize()`
+    * `opts`: options (see below)
+
+    Returns:
+    * `handler`: function to run the process. This should be run until the
+      first returned value is false. The remaining return values match `Deserialize()`.
+      `handler` returns:
+      * `success`: a boolean indicating if deserialization was successful
+      * `...`: the deserialized value(s), or a string containing the encountered Lua error
+
 * **`LibSerialize:IsSerializableType(...)`**
 
     Arguments:
@@ -182,6 +236,17 @@ The following serialization options are supported:
     table encountered during serialization. The function must return true for
     the pair to be serialized. It may be called multiple times on a table for
     the same key/value pair. See notes on reeentrancy and table modification.
+When using `SerializeAsyncEx()`, these additional options are supported:
+  * `yieldOnObjectCount`: `number` How large to allow the buffer before yielding
+  * `yieldOnElapsedTime`: `number` Max duration between yields
+  * `timeFn`: `function` To return the time in `number` format relevant to the
+    environment.
+
+The following deserialization options are supported with `DeserializeAsync`:
+  * `yieldOnObjectCount`: `number` How large to allow the buffer before yielding
+  * `yieldOnElapsedTime`: `number` Max duration between yields
+  * `timeFn`: `function` To return the time in `number` format relevant to the
+    environment.
 
 If an option is unspecified in the table, then its default will be used.
 This means that if an option `foo` defaults to true, then:
@@ -263,6 +328,29 @@ the following possible keys:
     assert(tab.nested.c == nil)
     ```
 
+5. `LibSerialize:SerializeAsync()` serializes data in a coroutine which
+    ease the stresses of some environments.
+    ```lua
+    local t = { "test", [false] = {} }
+    t[ t[false] ] = "hello"
+    local co_handler = LibSerialize:SerializeAsync(t, "extra")
+    local ongoing, serialized
+    repeat
+        ongoing, serialized = co_handler()
+    until not ongoing
+
+    local tab
+    co_handler = LibSerialize:DeserializeAsync(serialized)
+    repeat
+        ongoing, tab = co_handler()
+    until not ongoing
+
+    assert(success)
+    assert(tab[1] == "test")
+    assert(tab[ tab[false] ] == "hello")
+    assert(str == "extra")
+    ```
+
 
 ## Encoding format:
 Every object is encoded as a type byte followed by type-dependent payload.
@@ -308,7 +396,7 @@ The type byte uses the following formats to implement the above:
     * Followed by the type-dependent payload, including count(s) if needed
 --]]
 
-local MAJOR, MINOR = "LibSerialize", 4
+local MAJOR, MINOR = "LibSerialize", 5
 local LibSerialize
 if LibStub then
     LibSerialize = LibStub:NewLibrary(MAJOR, MINOR)
@@ -348,18 +436,28 @@ local string_sub = string.sub
 local table_concat = table.concat
 local table_insert = table.insert
 local table_sort = table.sort
+local coroutine_create = coroutine.create
+local coroutine_status = coroutine.status
+local coroutine_resume = coroutine.resume
+local coroutine_yield = coroutine.yield
 
-local defaultOptions = {
+local defaultSerializeOptions = {
     errorOnUnserializableType = true,
     stable = false,
-    filter = nil,
+    filter = nil
+}
+local defaultAsyncOptions = {
+    yieldOnObjectCount = 4096,
+    timeFn = nil,
+    yieldOnElapsedTime = nil
+}
+local defaultDeserializeOptions = {
 }
 
 local canSerializeFnOptions = {
     errorOnUnserializableType = false
 }
 
-
 --[[---------------------------------------------------------------------------
     Helper functions.
 --]]---------------------------------------------------------------------------
@@ -468,7 +566,7 @@ local function CreateWriter()
 
     -- Write the entire string into the writer.
     local function WriteString(str)
-        -- DebugPrint("Writing string:", str, #str)
+        -- DebugPrint("Writing string:", str, #str, bufferSize)
         bufferSize = bufferSize + 1
         buffer[bufferSize] = str
     end
@@ -632,7 +730,7 @@ end
 
 local LibSerializeInt = {}
 
-local function CreateSerializer(opts)
+local function CreateSerializer(opts, asyncMode)
     local state = {}
 
     -- Copy the state from LibSerializeInt.
@@ -650,17 +748,31 @@ local function CreateSerializer(opts)
     -- Create a combined options table, starting with the defaults
     -- and then overwriting any user-supplied keys.
     state._opts = {}
-    for k, v in pairs(defaultOptions) do
+    for k, v in pairs(defaultSerializeOptions) do
         state._opts[k] = v
     end
+    if asyncMode then
+        state._async = true
+        for k, v in pairs(defaultAsyncOptions) do
+            state._opts[k] = v
+        end
+    end
     for k, v in pairs(opts) do
         state._opts[k] = v
     end
 
+    -- Initialize yield counters
+    if state._opts.yieldOnObjectCount ~= false then
+        state._currentObjectCount = 0
+    end
+    if state._opts.timeFn and state._opts.yieldOnElapsedTime then
+        state._currentElapsedTime = opts.timeFn()
+    end
+
     return state
 end
 
-local function CreateDeserializer(input)
+local function CreateDeserializer(input, opts, asyncMode)
     local state = {}
 
     -- Copy the state from LibSerializeInt.
@@ -675,6 +787,30 @@ local function CreateDeserializer(input)
     -- Create the reader functions.
     state._readBytes, state._readerBytesLeft = CreateReader(input)
 
+    -- Create a combined options table, starting with the defaults
+    -- and then overwriting any user-supplied keys.
+    state._opts = {}
+    for k, v in pairs(defaultDeserializeOptions) do
+        state._opts[k] = v
+    end
+    if asyncMode then
+        state._async = true
+        for k, v in pairs(defaultAsyncOptions) do
+            state._opts[k] = v
+        end
+    end
+    for k, v in pairs(opts) do
+        state._opts[k] = v
+    end
+
+    -- Initialize yield counters
+    if state._opts.yieldOnObjectCount ~= false then
+        state._currentObjectCount = 0
+    end
+    if state._opts.timeFn and state._opts.yieldOnElapsedTime then
+        state._currentElapsedTime = opts.timeFn()
+    end
+
     return state
 end
 
@@ -700,6 +836,22 @@ end
 function LibSerializeInt:_ReadObject()
     local value = self:_ReadByte()
 
+    if self._async then
+        if self._currentObjectCount then
+            self._currentObjectCount = self._currentObjectCount + 1
+        end
+        local elapsedTime
+        if self._currentElapsedTime and self._opts.timeFn then
+            elapsedTime = self._opts.timeFn()
+        end
+        if (elapsedTime and elapsedTime - self._currentElapsedTime > self._opts.yieldOnElapsedTime) or
+        (self._currentObjectCount and self._currentObjectCount > self._opts.yieldOnObjectCount) then
+            if elapsedTime then self._currentElapsedTime = elapsedTime end
+            if self._currentObjectCount then self._currentObjectCount = 0 end
+            coroutine_yield()
+        end
+    end
+
     if value % 2 == 1 then
         -- Number embedded in the top 7 bits.
         local num = (value - 1) / 2
@@ -966,6 +1118,22 @@ end
 -- Note that _GetWriteFn will raise a Lua error if it finds an
 -- unserializable type, unless this behavior is suppressed via options.
 function LibSerializeInt:_WriteObject(obj)
+    if self._async then
+        if self._currentObjectCount then
+            self._currentObjectCount = self._currentObjectCount + 1
+        end
+        local elapsedTime
+        if self._currentElapsedTime and self._opts.timeFn then
+            elapsedTime = self._opts.timeFn()
+        end
+        if (elapsedTime and elapsedTime - self._currentElapsedTime > self._opts.yieldOnElapsedTime) or
+        (self._currentObjectCount and self._currentObjectCount > self._opts.yieldOnObjectCount) then
+            if elapsedTime then self._currentElapsedTime = elapsedTime end
+            if self._currentObjectCount then self._currentObjectCount = 0 end
+            coroutine_yield()
+        end
+    end
+
     local writeFn = self:_GetWriteFn(obj)
     if not writeFn then
         return false
@@ -1310,11 +1478,8 @@ function LibSerialize:IsSerializableType(...)
     return serializeTester:_CanSerialize(canSerializeFnOptions, ...)
 end
 
-function LibSerialize:SerializeEx(opts, ...)
-    local ser = CreateSerializer(opts)
-
+local function serializeOperation(ser, ...)
     ser:_WriteByte(SERIALIZATION_VERSION)
-
     for i = 1, select("#", ...) do
         local input = select(i, ...)
         if not ser:_WriteObject(input) then
@@ -1328,13 +1493,42 @@ function LibSerialize:SerializeEx(opts, ...)
     return ser._flushWriter()
 end
 
-function LibSerialize:Serialize(...)
-    return self:SerializeEx(defaultOptions, ...)
+function LibSerialize:SerializeEx(opts, ...)
+    local ser = CreateSerializer(opts)
+    return serializeOperation(ser, ...)
 end
 
-function LibSerialize:DeserializeValue(input)
-    local deser = CreateDeserializer(input)
+function LibSerialize:SerializeAsyncEx(opts, ...)
+    local ser = CreateSerializer(opts, true)
 
+    if opts.yieldOnElapsedTime and not opts.timeFn then
+        error("Async Mode operation with yieldOnElapsedTime requires timeFn option")
+    end
+
+    local thread = coroutine_create(serializeOperation)
+    local dots = {...}
+    -- return coroutine handler
+    return function()
+        local co_success, result = coroutine_resume(thread, ser, unpack(dots))
+        if not co_success then
+            error(result)
+        elseif coroutine_status(thread) ~= 'dead' then
+            return true
+        else
+            return false, result
+        end
+    end
+end
+
+function LibSerialize:SerializeAsync(...)
+    return self:SerializeAsyncEx(defaultAsyncOptions, ...)
+end
+
+function LibSerialize:Serialize(...)
+    return self:SerializeEx(defaultSerializeOptions, ...)
+end
+
+local function deserializeOperation(deser)
     -- Since there's only one compression version currently,
     -- no extra work needs to be done to decode the data.
     local version = deser:_ReadByte()
@@ -1358,8 +1552,39 @@ function LibSerialize:DeserializeValue(input)
     return unpack(output, 1, outputSize)
 end
 
+function LibSerialize:DeserializeValue(input, opts)
+    local deser = CreateDeserializer(input, opts or {})
+    return deserializeOperation(deser)
+end
+
+function LibSerialize:DeserializeAsyncValue(input, opts)
+    if opts.yieldOnElapsedTime and not opts.timeFn then
+        error("Async Mode operation with yieldOnElapsedTime requires timeFn option")
+    end
+
+    local deser = CreateDeserializer(input, opts or {}, true)
+
+    local thread = coroutine_create(deserializeOperation)
+    -- return coroutine handler
+    return function()
+        local values = {coroutine_resume(thread, deser)}
+        if not values[1] then
+            error(values[2])
+        elseif coroutine_status(thread) ~= 'dead' then
+            return true
+        else
+            return false, unpack(values)
+        end
+    end
+
+end
+
 function LibSerialize:Deserialize(input)
     return pcall(self.DeserializeValue, self, input)
 end
 
+function LibSerialize:DeserializeAsync(input, opts)
+    return self:DeserializeAsyncValue(input, opts)
+end
+
 return LibSerialize