From 60ff38bd31c80f1e551df2f252e99822f45bdbf8 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 24 Oct 2019 10:34:13 +0800
Subject: [PATCH 1/5] Fix reorder2default

Change-Id: I74c87af9535f6264e6d1ea7eaed089a6480a3358
---
 src/ndarray/ndarray.cc | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index e15f72fa6cfa..7a86681e484d 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -1638,8 +1638,9 @@ void NDArray::Save(dmlc::Stream *strm) const {
     this->WaitToRead();
     nd_cpu = *this;
 #if MXNET_USE_MKLDNN == 1
-    if (nd_cpu.IsMKLDNNData())
-      nd_cpu = nd_cpu.Reorder2Default();
+    // For mkldnn, a copy of *this can ensure no write access pending on *this.
+    nd_cpu = this->Copy(Context::CPU());
+    nd_cpu.WaitToRead();
 #endif
     save_data = nd_cpu.data();
   }
@@ -2023,15 +2024,17 @@ void NDArray::SyncCopyToCPU(void *data, size_t size) const {
   TBlob dst(data, dshape, cpu::kDevMask, this->dtype_, 0); // NOLINT(*)
 
   if (this->ctx().dev_mask() == cpu::kDevMask) {
-    this->WaitToRead();
-    RunContext rctx{this->ctx(), nullptr, nullptr, false};
-    NDArray src = *this;
+    Engine::Get()->PushAsync(
+        [&](RunContext rctx, Engine::CallbackOnComplete on_complete) {
+          NDArray src = *this;
 #if MXNET_USE_MKLDNN == 1
-    if (src.IsMKLDNNData())
-      src = this->Reorder2Default();
+          src = this->Reorder2Default();
 #endif
-    ndarray::Copy<cpu, cpu>(src.data(), &dst,
-                            Context::CPU(), Context::CPU(), rctx);
+          ndarray::Copy<cpu, cpu>(src.data(), &dst, Context::CPU(), Context::CPU(), rctx);
+          on_complete();
+        },
+        this->ctx(), {this->var()}, {}, FnProperty::kNormal, 0, "Reorder2Default");
+    this->WaitToWrite();
   } else {
 #if MXNET_USE_CUDA
     Engine::Get()->PushAsync(

From 2307bc3383e32cb7fcdb3652ea650e0830882ce3 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 24 Oct 2019 10:50:59 +0800
Subject: [PATCH 2/5] fix

Change-Id: I6d07b43b520a47e7c78bd4b4b6390f5fb95e6957
---
 src/ndarray/ndarray.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 7a86681e484d..1056dc19d444 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -1635,7 +1635,6 @@ void NDArray::Save(dmlc::Stream *strm) const {
     nd_cpu.WaitToRead();
     save_data = nd_cpu.data();
   } else {
-    this->WaitToRead();
     nd_cpu = *this;
 #if MXNET_USE_MKLDNN == 1
     // For mkldnn, a copy of *this can ensure no write access pending on *this.

From 312184191df92b989d01e589a54d1dfceccef2cf Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 24 Oct 2019 12:26:55 +0800
Subject: [PATCH 3/5] Fix

Change-Id: Id72f25c34291be4711f55569c6d61467edd6113d
---
 src/ndarray/ndarray.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 1056dc19d444..7ee3f4b84492 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -2025,14 +2025,15 @@ void NDArray::SyncCopyToCPU(void *data, size_t size) const {
   if (this->ctx().dev_mask() == cpu::kDevMask) {
     Engine::Get()->PushAsync(
         [&](RunContext rctx, Engine::CallbackOnComplete on_complete) {
+          RunContext ctx{this->ctx(), nullptr, nullptr, false};
           NDArray src = *this;
 #if MXNET_USE_MKLDNN == 1
           src = this->Reorder2Default();
 #endif
-          ndarray::Copy<cpu, cpu>(src.data(), &dst, Context::CPU(), Context::CPU(), rctx);
+          ndarray::Copy<cpu, cpu>(src.data(), &dst, Context::CPU(), Context::CPU(), ctx);
           on_complete();
         },
-        this->ctx(), {this->var()}, {}, FnProperty::kNormal, 0, "Reorder2Default");
+        this->ctx(), {this->var()}, {}, FnProperty::kNormal, 0, "SyncCopyCPU2CPU");
     this->WaitToWrite();
   } else {
 #if MXNET_USE_CUDA

From cf02d9be6c197e612f6d07c0c924d37ea46b0d6d Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 24 Oct 2019 14:24:33 +0800
Subject: [PATCH 4/5] Fix CI

Change-Id: I8c33a82555d5ace2d0b682c1e3eefa13f3a44768
---
 src/ndarray/ndarray.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 7ee3f4b84492..f807f2d6b5f7 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -1635,11 +1635,13 @@ void NDArray::Save(dmlc::Stream *strm) const {
     nd_cpu.WaitToRead();
     save_data = nd_cpu.data();
   } else {
-    nd_cpu = *this;
 #if MXNET_USE_MKLDNN == 1
     // For mkldnn, a copy of *this can ensure no write access pending on *this.
     nd_cpu = this->Copy(Context::CPU());
     nd_cpu.WaitToRead();
+#else
+    this->WaitToRead();
+    nd_cpu = *this;
 #endif
     save_data = nd_cpu.data();
   }

From 5b764915955b2a91bb0dc2f285a99b8ee38f0277 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 24 Oct 2019 14:58:51 +0800
Subject: [PATCH 5/5] Run CI

Change-Id: Ie8a6dab80ef91c0337cafbae4e3db277e0c7ebf7