From 7f6a8214612640abf77f79d34bd333ac6b355caa Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 15:57:50 -0700
Subject: [PATCH 01/13] use threading for mp dataloader fetching, allow
 pin_memory option

---
 python/mxnet/gluon/data/dataloader.py | 48 +++++++++++++++++++++------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index eb1eb419cd02..9f290222f6c1 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -26,6 +26,7 @@
 import multiprocessing
 import multiprocessing.queues
 from multiprocessing.reduction import ForkingPickler
+import threading
 import numpy as np
 
 try:
@@ -159,9 +160,23 @@ def worker_loop(dataset, key_queue, data_queue, batchify_fn):
         batch = batchify_fn([dataset[i] for i in samples])
         data_queue.put((idx, batch))
 
+def fetcher_loop(data_queue, data_buffer, pin_memory=False):
+    """Fetcher loop for fetching data from queue and put in reorder dict."""
+    while True:
+        idx, batch = data_queue.get()
+        if idx is None:
+            break
+        if pin_memory:
+            try:
+                batch = batch.as_in_context(context.cpu_pinned())
+            except:
+                pass
+        batch = batch.as_in_context(context.cpu())
+        data_buffer[idx] = batch
+
 class _MultiWorkerIter(object):
     """Interal multi-worker iterator for DataLoader."""
-    def __init__(self, num_workers, dataset, batchify_fn, batch_sampler):
+    def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory=False):
         assert num_workers > 0, "_MultiWorkerIter is not for {} workers".format(num_workers)
         self._num_workers = num_workers
         self._dataset = dataset
@@ -184,6 +199,12 @@ def __init__(self, num_workers, dataset, batchify_fn, batch_sampler):
             worker.start()
             workers.append(worker)
 
+        self._fetcher = threading.Thread(
+            target=fetcher_loop,
+            args=(self._data_queue, self._data_buffer, pin_memory))
+        self._fetcher.daemon = True
+        self._fetcher.start()
+
         # pre-fetch
         for _ in range(2 * self._num_workers):
             self._push_next()
@@ -210,13 +231,11 @@ def __next__(self):
             raise StopIteration
 
         while True:
-            self._push_next()
             if self._rcvd_idx in self._data_buffer:
                 batch = self._data_buffer.pop(self._rcvd_idx)
                 self._rcvd_idx += 1
+                self._push_next()
                 return batch
-            idx, batch = self._data_queue.get()
-            self._data_buffer[idx] = batch
 
     def next(self):
         return self.__next__()
@@ -234,6 +253,7 @@ def shutdown(self):
                     self._data_queue.get()
             except IOError:
                 pass
+            self._data_queue.put((None, None))
             self._shutdown = True
 
 
@@ -277,12 +297,16 @@ def default_batchify_fn(data):
 
     num_workers : int, default 0
         The number of multiprocessing workers to use for data preprocessing.
-        `num_workers > 0` is not supported on Windows yet.
+    pin_memory : boolean, default False
+        If ``True``, the dataloader will copy NDArrays into pinned memory
+        before returning them. Copying from CPU pinned memory to GPU is faster
+        than from normal CPU memory.
     """
     def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None,
                  last_batch=None, batch_sampler=None, batchify_fn=None,
-                 num_workers=0):
+                 num_workers=0, pin_memory=False):
         self._dataset = dataset
+        self._pin_memory = pin_memory
 
         if batch_sampler is None:
             if batch_size is None:
@@ -315,13 +339,17 @@ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None,
 
     def __iter__(self):
         if self._num_workers == 0:
-            generator = lambda: [(yield self._batchify_fn([self._dataset[idx] for idx in batch]))
-                                 for batch in self._batch_sampler]
-            return generator()
+            def same_process_iter():
+                for batch in self._batch_sampler:
+                    ret = self._batchify_fn([self._dataset[idx] for idx in batch])
+                    if self._pin_memory:
+                        ret = ret.as_in_context(context.cpu_pinned())
+                    yield ret
+            return same_process_iter()
 
         # multi-worker
         return _MultiWorkerIter(self._num_workers, self._dataset,
-                                self._batchify_fn, self._batch_sampler)
+                                self._batchify_fn, self._batch_sampler, self._pin_memory)
 
     def __len__(self):
         return len(self._batch_sampler)

From 80c13688e20e982202e9c850a601342cdb190f5b Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 16:47:35 -0700
Subject: [PATCH 02/13] allow pin tuple of data into cpu_pinned

---
 python/mxnet/gluon/data/dataloader.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 9f290222f6c1..dc574e9311cf 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -150,6 +150,14 @@ def default_mp_batchify_fn(data):
                         ctx=context.Context('cpu_shared', 0))
 
 
+def pin_memory_batch(data):
+    """Move data into cpu_pinned memory. """
+    if isinstance(data, nd.NDArray):
+        return data.as_in_context(context.cpu_pinned())
+    elif isinstance(data, (list, tuple)):
+        return [pin_memory_batch(d) for d in data]
+    return data
+
 def worker_loop(dataset, key_queue, data_queue, batchify_fn):
     """Worker loop for multiprocessing DataLoader."""
     dataset._fork()
@@ -171,7 +179,7 @@ def fetcher_loop(data_queue, data_buffer, pin_memory=False):
                 batch = batch.as_in_context(context.cpu_pinned())
             except:
                 pass
-        batch = batch.as_in_context(context.cpu())
+        batch = pin_memory_batch(batch)
         data_buffer[idx] = batch
 
 class _MultiWorkerIter(object):
@@ -343,7 +351,7 @@ def same_process_iter():
                 for batch in self._batch_sampler:
                     ret = self._batchify_fn([self._dataset[idx] for idx in batch])
                     if self._pin_memory:
-                        ret = ret.as_in_context(context.cpu_pinned())
+                        ret = pin_memory_batch(ret)
                     yield ret
             return same_process_iter()
 

From e49bb448555f77ac65d6028737cee2b6727a6b20 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 17:22:19 -0700
Subject: [PATCH 03/13] fix as_in_context if not cpu_pinned

---
 python/mxnet/gluon/data/dataloader.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index dc574e9311cf..7a6a80cf832e 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -150,12 +150,12 @@ def default_mp_batchify_fn(data):
                         ctx=context.Context('cpu_shared', 0))
 
 
-def pin_memory_batch(data):
-    """Move data into cpu_pinned memory. """
+def _as_in_context(data, ctx):
+    """Move data into new context. """
     if isinstance(data, nd.NDArray):
-        return data.as_in_context(context.cpu_pinned())
+        return data.as_in_context(ctx)
     elif isinstance(data, (list, tuple)):
-        return [pin_memory_batch(d) for d in data]
+        return [_as_in_context(d, ctx) for d in data]
     return data
 
 def worker_loop(dataset, key_queue, data_queue, batchify_fn):
@@ -176,10 +176,10 @@ def fetcher_loop(data_queue, data_buffer, pin_memory=False):
             break
         if pin_memory:
             try:
-                batch = batch.as_in_context(context.cpu_pinned())
+                batch = _as_in_context(batch, context.cpu_pinned())
             except:
                 pass
-        batch = pin_memory_batch(batch)
+        batch = _as_in_context(batch, context.cpu())
         data_buffer[idx] = batch
 
 class _MultiWorkerIter(object):
@@ -351,7 +351,7 @@ def same_process_iter():
                 for batch in self._batch_sampler:
                     ret = self._batchify_fn([self._dataset[idx] for idx in batch])
                     if self._pin_memory:
-                        ret = pin_memory_batch(ret)
+                        ret = _as_in_context(ret, context.cpu_shared())
                     yield ret
             return same_process_iter()
 

From 2642f582db243ea6507edcd718e3589213f6ee30 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 17:25:10 -0700
Subject: [PATCH 04/13] fix cpu_pinned

---
 python/mxnet/gluon/data/dataloader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 7a6a80cf832e..d5976a9caf31 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -351,7 +351,7 @@ def same_process_iter():
                 for batch in self._batch_sampler:
                     ret = self._batchify_fn([self._dataset[idx] for idx in batch])
                     if self._pin_memory:
-                        ret = _as_in_context(ret, context.cpu_shared())
+                        ret = _as_in_context(ret, context.cpu_pinned())
                     yield ret
             return same_process_iter()
 

From b7481f9fdc8466273607635ad59046912e1f179b Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 17:49:18 -0700
Subject: [PATCH 05/13] fix unittest for windows, update doc that windows mp is
 available

---
 python/mxnet/gluon/data/dataloader.py    |   2 +-
 tests/python/unittest/test_gluon_data.py | 169 +++++++++++------------
 2 files changed, 82 insertions(+), 89 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index d5976a9caf31..13f16527898b 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -151,7 +151,7 @@ def default_mp_batchify_fn(data):
 
 
 def _as_in_context(data, ctx):
-    """Move data into new context. """
+    """Move data into new context."""
     if isinstance(data, nd.NDArray):
         return data.as_in_context(ctx)
     elif isinstance(data, (list, tuple)):
diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py
index 043804487b5e..37e6364aabc9 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -130,99 +130,92 @@ def test_multi_worker():
     for i, batch in enumerate(loader):
         assert (batch.asnumpy() == i).all()
 
+class _Dummy(Dataset):
+    """Dummpy dataset for randomized shape arrays."""
+    def __init__(self, random_shape):
+        self.random_shape = random_shape
+
+    def __getitem__(self, idx):
+        key = idx
+        if self.random_shape:
+            out = np.random.uniform(size=(random.randint(1000, 1100), 40))
+            labels = np.random.uniform(size=(random.randint(10, 15)))
+        else:
+            out = np.random.uniform(size=(1000, 40))
+            labels = np.random.uniform(size=(10))
+        return key, out, labels
+
+    def __len__(self):
+        return 50
+
+def _batchify_list(data):
+    """
+    return list of ndarray without stack/concat/pad
+    """
+    if isinstance(data, (tuple, list)):
+        return list(data)
+    if isinstance(data, mx.nd.NDArray):
+        return [data]
+    return data
+
+def _batchify(data):
+    """
+    Collate data into batch. Use shared memory for stacking.
+    :param data: a list of array, with layout of 'NTC'.
+    :return either x  and x's unpadded lengths, or x, x's unpadded lengths, y and y's unpadded lengths
+            if labels are not supplied.
+    """
+
+    # input layout is NTC
+    keys, inputs, labels = [item[0] for item in data], [item[1] for item in data], \
+                           [item[2] for item in data]
+
+    if len(data) > 1:
+        max_data_len = max([seq.shape[0] for seq in inputs])
+        max_labels_len = 0 if not labels else max([seq.shape[0] for seq in labels])
+    else:
+        max_data_len = inputs[0].shape[0]
+        max_labels_len = 0 if not labels else labels[0].shape[0]
+
+    x_lens = [item.shape[0] for item in inputs]
+    y_lens = [item.shape[0] for item in labels]
+
+    for i, seq in enumerate(inputs):
+        pad_len = max_data_len - seq.shape[0]
+        inputs[i] = np.pad(seq, ((0, pad_len), (0, 0)), 'constant', constant_values=0)
+        labels[i] = np.pad(labels[i], (0, max_labels_len - labels[i].shape[0]),
+                           'constant', constant_values=-1)
+
+    inputs = np.asarray(inputs, dtype=np.float32)
+    if labels is not None:
+        labels = np.asarray(labels, dtype=np.float32)
+    inputs = inputs.transpose((1, 0, 2))
+    labels = labels.transpose((1, 0))
+
+    return (nd.array(inputs, dtype=inputs.dtype, ctx=context.Context('cpu_shared', 0)),
+            nd.array(x_lens, ctx=context.Context('cpu_shared', 0))) \
+        if labels is None else (
+        nd.array(inputs, dtype=inputs.dtype, ctx=context.Context('cpu_shared', 0)),
+        nd.array(x_lens, ctx=context.Context('cpu_shared', 0)),
+        nd.array(labels, dtype=labels.dtype, ctx=context.Context('cpu_shared', 0)),
+        nd.array(y_lens, ctx=context.Context('cpu_shared', 0)))
+
 @with_seed()
 def test_multi_worker_forked_data_loader():
     """
     Test should successfully run its course of multi-process/forked data loader without errors
     """
-    class Dummy(Dataset):
-        def __init__(self, random_shape):
-            self.random_shape = random_shape
-
-        def __getitem__(self, idx):
-            key = idx
-            if self.random_shape:
-                out = np.random.uniform(size=(random.randint(1000, 1100), 40))
-                labels = np.random.uniform(size=(random.randint(10, 15)))
-            else:
-                out = np.random.uniform(size=(1000, 40))
-                labels = np.random.uniform(size=(10))
-            return key, out, labels
-
-        def __len__(self):
-            return 50
-
-        def batchify_list(self, data):
-            """
-            return list of ndarray without stack/concat/pad
-            """
-            if isinstance(data, (tuple, list)):
-                return list(data)
-            if isinstance(data, mx.nd.NDArray):
-                return [data]
-            return data
-
-        def batchify(self, data):
-            """
-            Collate data into batch. Use shared memory for stacking.
-
-            :param data: a list of array, with layout of 'NTC'.
-            :return either x  and x's unpadded lengths, or x, x's unpadded lengths, y and y's unpadded lengths
-                    if labels are not supplied.
-            """
-
-            # input layout is NTC
-            keys, inputs, labels = [item[0] for item in data], [item[1] for item in data], \
-                                   [item[2] for item in data]
-
-            if len(data) > 1:
-                max_data_len = max([seq.shape[0] for seq in inputs])
-                max_labels_len = 0 if not labels else max([seq.shape[0] for seq in labels])
-            else:
-                max_data_len = inputs[0].shape[0]
-                max_labels_len = 0 if not labels else labels[0].shape[0]
-
-            x_lens = [item.shape[0] for item in inputs]
-            y_lens = [item.shape[0] for item in labels]
-
-            for i, seq in enumerate(inputs):
-                pad_len = max_data_len - seq.shape[0]
-                inputs[i] = np.pad(seq, ((0, pad_len), (0, 0)), 'constant', constant_values=0)
-                labels[i] = np.pad(labels[i], (0, max_labels_len - labels[i].shape[0]),
-                                   'constant', constant_values=-1)
-
-            inputs = np.asarray(inputs, dtype=np.float32)
-            if labels is not None:
-                labels = np.asarray(labels, dtype=np.float32)
-            inputs = inputs.transpose((1, 0, 2))
-            labels = labels.transpose((1, 0))
-
-            return (nd.array(inputs, dtype=inputs.dtype, ctx=context.Context('cpu_shared', 0)),
-                    nd.array(x_lens, ctx=context.Context('cpu_shared', 0))) \
-                if labels is None else (
-                nd.array(inputs, dtype=inputs.dtype, ctx=context.Context('cpu_shared', 0)),
-                nd.array(x_lens, ctx=context.Context('cpu_shared', 0)),
-                nd.array(labels, dtype=labels.dtype, ctx=context.Context('cpu_shared', 0)),
-                nd.array(y_lens, ctx=context.Context('cpu_shared', 0)))
-
-
-    # This test is pointless on Windows because Windows doesn't fork
-    if platform.system() != 'Windows':
-        data = Dummy(True)
-        loader = DataLoader(data, batch_size=40, batchify_fn=data.batchify, num_workers=2)
-        for epoch in range(1):
-            for i, data in enumerate(loader):
-                if i % 100 == 0:
-                    print(data)
-                    print('{}:{}'.format(epoch, i))
-
-        data = Dummy(True)
-        loader = DataLoader(data, batch_size=40, batchify_fn=data.batchify_list, num_workers=2)
-        for epoch in range(1):
-            for i, data in enumerate(loader):
-                if i % 100 == 0:
-                    print(data)
-                    print('{}:{}'.format(epoch, i))
+    data = _Dummy(False)
+    loader = DataLoader(data, batch_size=40, batchify_fn=_batchify, num_workers=2)
+    for epoch in range(1):
+        for i, data in enumerate(loader):
+            pass
+
+    data = _Dummy(True)
+    loader = DataLoader(data, batch_size=40, batchify_fn=_batchify_list, num_workers=2)
+    for epoch in range(1):
+        for i, data in enumerate(loader):
+            pass
 
 if __name__ == '__main__':
     import nose

From 61ddac4d1591e476161dff0b1fa0e8c22164cabf Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 17:57:33 -0700
Subject: [PATCH 06/13] fix pin_memory

---
 python/mxnet/gluon/data/dataloader.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 13f16527898b..8282bb3593a1 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -175,11 +175,9 @@ def fetcher_loop(data_queue, data_buffer, pin_memory=False):
         if idx is None:
             break
         if pin_memory:
-            try:
-                batch = _as_in_context(batch, context.cpu_pinned())
-            except:
-                pass
-        batch = _as_in_context(batch, context.cpu())
+            batch = _as_in_context(batch, context.cpu_pinned())
+        else:
+            batch = _as_in_context(batch, context.cpu())
         data_buffer[idx] = batch
 
 class _MultiWorkerIter(object):

From 14f519295b180bb077d637e9803def09abbf9e42 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Thu, 26 Jul 2018 18:43:17 -0700
Subject: [PATCH 07/13] fix lint

---
 python/mxnet/gluon/data/dataloader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 8282bb3593a1..a5038785444c 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -16,7 +16,7 @@
 # under the License.
 
 # coding: utf-8
-# pylint: disable=
+# pylint: disable=ungrouped-imports
 """Dataset generator."""
 __all__ = ['DataLoader']
 

From 91a51022e8b532cfb4de910be6186ed0e0dc507d Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Fri, 27 Jul 2018 23:18:49 -0700
Subject: [PATCH 08/13] always use simplequeue for data queue

---
 python/mxnet/gluon/data/dataloader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index a5038785444c..6795612871f7 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -189,7 +189,7 @@ def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory=
         self._batchify_fn = batchify_fn
         self._batch_sampler = batch_sampler
         self._key_queue = Queue()
-        self._data_queue = Queue() if sys.version_info[0] <= 2 else SimpleQueue()
+        self._data_queue = SimpleQueue()
         self._data_buffer = {}
         self._rcvd_idx = 0
         self._sent_idx = 0

From c37e3b7e7b20a704add5d46a221ad84656a3b4a6 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Mon, 30 Jul 2018 15:43:50 -0700
Subject: [PATCH 09/13] remove main thread clearing for data_queue

---
 python/mxnet/gluon/data/dataloader.py    | 7 +------
 tests/python/unittest/test_gluon_data.py | 3 ---
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 6795612871f7..13ab544a03d3 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -189,7 +189,7 @@ def __init__(self, num_workers, dataset, batchify_fn, batch_sampler, pin_memory=
         self._batchify_fn = batchify_fn
         self._batch_sampler = batch_sampler
         self._key_queue = Queue()
-        self._data_queue = SimpleQueue()
+        self._data_queue = Queue() if sys.version_info[0] <= 2 else SimpleQueue()
         self._data_buffer = {}
         self._rcvd_idx = 0
         self._sent_idx = 0
@@ -254,11 +254,6 @@ def shutdown(self):
         if not self._shutdown:
             for _ in range(self._num_workers):
                 self._key_queue.put((None, None))
-            try:
-                while not self._data_queue.empty():
-                    self._data_queue.get()
-            except IOError:
-                pass
             self._data_queue.put((None, None))
             self._shutdown = True
 
diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py
index 37e6364aabc9..dea044dcd513 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -202,9 +202,6 @@ def _batchify(data):
 
 @with_seed()
 def test_multi_worker_forked_data_loader():
-    """
-    Test should successfully run its course of multi-process/forked data loader without errors
-    """
     data = _Dummy(False)
     loader = DataLoader(data, batch_size=40, batchify_fn=_batchify, num_workers=2)
     for epoch in range(1):

From 52f96eb6219f70c47f42b607a5dfacee9182d032 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Mon, 6 Aug 2018 15:37:09 -0700
Subject: [PATCH 10/13] do not use outside folder as pythonpath but run
 nosetests inside

---
 ci/windows/test_py2_cpu.ps1 | 3 ++-
 ci/windows/test_py2_gpu.ps1 | 3 ++-
 ci/windows/test_py3_cpu.ps1 | 3 ++-
 ci/windows/test_py3_gpu.ps1 | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1
index 1623d2956103..eb0b1a8affd7 100644
--- a/ci/windows/test_py2_cpu.ps1
+++ b/ci/windows/test_py2_cpu.ps1
@@ -16,7 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
+$env:PYTHONPATH=join-path $pwd.Path python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1
index 13cd5366e0db..7ffe6482e2bb 100644
--- a/ci/windows/test_py2_gpu.ps1
+++ b/ci/windows/test_py2_gpu.ps1
@@ -16,7 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
+$env:PYTHONPATH=join-path $pwd.Path python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
index 98d4e410e8f5..ec5d569a16d3 100644
--- a/ci/windows/test_py3_cpu.ps1
+++ b/ci/windows/test_py3_cpu.ps1
@@ -16,7 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
+$env:PYTHONPATH=join-path $pwd.Path python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
index b94b4f389be8..0fadcdb32825 100644
--- a/ci/windows/test_py3_gpu.ps1
+++ b/ci/windows/test_py3_gpu.ps1
@@ -16,7 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
+$env:PYTHONPATH=join-path $pwd.Path python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest

From 945b3d109cc1658256db3b08ef617e1991221d12 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Tue, 7 Aug 2018 15:57:22 -0700
Subject: [PATCH 11/13] use :MXNET_LIBRARY_PATH= to locate dll

---
 ci/windows/test_py2_cpu.ps1              | 4 ++--
 ci/windows/test_py2_gpu.ps1              | 4 ++--
 ci/windows/test_py3_cpu.ps1              | 4 ++--
 ci/windows/test_py3_gpu.ps1              | 4 ++--
 tests/python/unittest/test_gluon_data.py | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1
index eb0b1a8affd7..8333443ebd71 100644
--- a/ci/windows/test_py2_cpu.ps1
+++ b/ci/windows/test_py2_cpu.ps1
@@ -16,8 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
-$env:PYTHONPATH=join-path $pwd.Path python
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1
index 7ffe6482e2bb..f1d09db7fba4 100644
--- a/ci/windows/test_py2_gpu.ps1
+++ b/ci/windows/test_py2_gpu.ps1
@@ -16,8 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
-$env:PYTHONPATH=join-path $pwd.Path python
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
index ec5d569a16d3..fbde310c64ab 100644
--- a/ci/windows/test_py3_cpu.ps1
+++ b/ci/windows/test_py3_cpu.ps1
@@ -16,8 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
-$env:PYTHONPATH=join-path $pwd.Path python
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
index 0fadcdb32825..9def3905c506 100644
--- a/ci/windows/test_py3_gpu.ps1
+++ b/ci/windows/test_py3_gpu.ps1
@@ -16,8 +16,8 @@
 # under the License.
 
 7z x -y windows_package.7z
-for /r %%i in (*.dll) do xcopy /Y /C "%%i" .\\python\\mxnet\\
-$env:PYTHONPATH=join-path $pwd.Path python
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
 c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py
index dea044dcd513..4dc4f3ac8819 100644
--- a/tests/python/unittest/test_gluon_data.py
+++ b/tests/python/unittest/test_gluon_data.py
@@ -131,7 +131,7 @@ def test_multi_worker():
         assert (batch.asnumpy() == i).all()
 
 class _Dummy(Dataset):
-    """Dummpy dataset for randomized shape arrays."""
+    """Dummy dataset for randomized shape arrays."""
     def __init__(self, random_shape):
         self.random_shape = random_shape
 

From d06e631b356c57c49d664c204bfb4ad1416c5162 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Tue, 7 Aug 2018 17:08:44 -0700
Subject: [PATCH 12/13] fix dll path

---
 ci/windows/test_py2_cpu.ps1 | 2 +-
 ci/windows/test_py2_gpu.ps1 | 2 +-
 ci/windows/test_py3_cpu.ps1 | 2 +-
 ci/windows/test_py3_gpu.ps1 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1
index 8333443ebd71..ec19fd43900d 100644
--- a/ci/windows/test_py2_cpu.ps1
+++ b/ci/windows/test_py2_cpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1
index f1d09db7fba4..201a7d6490a4 100644
--- a/ci/windows/test_py2_gpu.ps1
+++ b/ci/windows/test_py2_gpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
index fbde310c64ab..f3266c08f622 100644
--- a/ci/windows/test_py3_cpu.ps1
+++ b/ci/windows/test_py3_cpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
index 9def3905c506..6cdf31046156 100644
--- a/ci/windows/test_py3_gpu.ps1
+++ b/ci/windows/test_py3_gpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\python\mxnet\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt

From b06638f36aa04756a4cdb696222ef23183aa1426 Mon Sep 17 00:00:00 2001
From: "Joshua Z. Zhang" <cheungchih@gmail.com>
Date: Wed, 8 Aug 2018 11:36:51 -0700
Subject: [PATCH 13/13] correct dll path

---
 ci/windows/test_py2_cpu.ps1 | 2 +-
 ci/windows/test_py2_gpu.ps1 | 2 +-
 ci/windows/test_py3_cpu.ps1 | 2 +-
 ci/windows/test_py3_gpu.ps1 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1
index ec19fd43900d..aa38b81e392e 100644
--- a/ci/windows/test_py2_cpu.ps1
+++ b/ci/windows/test_py2_cpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1
index 201a7d6490a4..5f8de5ac4f9e 100644
--- a/ci/windows/test_py2_gpu.ps1
+++ b/ci/windows/test_py2_gpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
index f3266c08f622..0dd48de26b32 100644
--- a/ci/windows/test_py3_cpu.ps1
+++ b/ci/windows/test_py3_cpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
index 6cdf31046156..4a0feb1ede88 100644
--- a/ci/windows/test_py3_gpu.ps1
+++ b/ci/windows/test_py3_gpu.ps1
@@ -16,7 +16,7 @@
 # under the License.
 
 7z x -y windows_package.7z
-$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\build\libmxnet.dll
+$env:MXNET_LIBRARY_PATH=join-path $pwd.Path windows_package\lib\libmxnet.dll
 $env:PYTHONPATH=join-path $pwd.Path windows_package\python
 $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
 c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt