From d43e9c4df6984362c53a0ea73d6205bc2bd69503 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Mon, 6 Aug 2018 18:53:31 -0700 Subject: [PATCH 1/9] reorder inputs --- src/operator/nn/mkldnn/mkldnn_base.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc index 4e4982e96ee5..99a655fd1dd4 100644 --- a/src/operator/nn/mkldnn/mkldnn_base.cc +++ b/src/operator/nn/mkldnn/mkldnn_base.cc @@ -473,8 +473,10 @@ void OpCheck::Init(const std::vector &inputs_, auto ctx = inputs_[0].ctx(); CHECK(!MKLDNNStream::Get()->HasOps()); for (size_t i = 0; i < inputs_.size(); i++) { - inputs.emplace_back(inputs_[i].shape(), ctx, - false, inputs_[i].dtype()); + NDArray data = inputs_[i]; + inputs.emplace_back(data.shape(), ctx, false, data.dtype()); + if (data.IsMKLDNNData() && data.IsView()) + data = in_data[fullc::kData].Reorder2Default(); auto mem = inputs_[i].GetMKLDNNData(); inputs[i].CopyFrom(*mem); } From 9cf9fecddb63c807d84d438301414c090b69d169 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Mon, 6 Aug 2018 19:17:22 -0700 Subject: [PATCH 2/9] use function flatten vs build in method --- python/mxnet/gluon/nn/basic_layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index ad69d4e9dd90..d26841977ac2 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -427,7 +427,7 @@ def __init__(self, **kwargs): super(Flatten, self).__init__(**kwargs) def hybrid_forward(self, F, x): - return x.reshape((0, -1)) + return F.Flatten(x) def __repr__(self): return self.__class__.__name__ From 8c797b325c2ed031d39be8b8d4d453b18998ab5d Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Tue, 7 Aug 2018 12:05:11 -0700 Subject: [PATCH 3/9] update similar array atoi to 0.01 --- src/operator/nn/mkldnn/mkldnn_base.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc index 99a655fd1dd4..7d4544c993b3 100644 --- a/src/operator/nn/mkldnn/mkldnn_base.cc +++ b/src/operator/nn/mkldnn/mkldnn_base.cc @@ -511,7 +511,7 @@ void OpCheck::Run(mxnet::FCompute fn, const nnvm::NodeAttrs &attrs, if (req[i] == kNullOp) continue; MSHADOW_TYPE_SWITCH(outputs[i].dtype(), DType, { - bool similar = SimilarArray(outputs[i], outputs_[i], 1e-3, 1e-3); + bool similar = SimilarArray(outputs[i], outputs_[i], 1e-2, 1e-2); if (!similar) { LOG(ERROR) << attrs.op->name << " fails"; } From 92c268b4fba858690b8a18b35e37d9b835f57c50 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Tue, 7 Aug 2018 12:14:53 -0700 Subject: [PATCH 4/9] fix reorder --- src/operator/nn/mkldnn/mkldnn_base.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc index 7d4544c993b3..8ad42a46c1a0 100644 --- a/src/operator/nn/mkldnn/mkldnn_base.cc +++ b/src/operator/nn/mkldnn/mkldnn_base.cc @@ -476,8 +476,8 @@ void OpCheck::Init(const std::vector &inputs_, NDArray data = inputs_[i]; inputs.emplace_back(data.shape(), ctx, false, data.dtype()); if (data.IsMKLDNNData() && data.IsView()) - data = in_data[fullc::kData].Reorder2Default(); - auto mem = inputs_[i].GetMKLDNNData(); + data = data.Reorder2Default(); + auto mem = data.GetMKLDNNData(); inputs[i].CopyFrom(*mem); } for (size_t i = 0; i < outputs_.size(); i++) { From db6f1cd9aee8f746f6c4c45db62f11151a208bed Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Wed, 8 Aug 2018 12:26:45 -0700 Subject: [PATCH 5/9] enable MXNET_MKLDNN_DEBUG in CI --- ci/docker/runtime_functions.sh | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1c861beb916c..5d876795ba33 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -527,9 +527,7 @@ sanity_check() { unittest_ubuntu_python2_cpu() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train @@ -539,9 +537,7 @@ unittest_ubuntu_python2_cpu() { unittest_ubuntu_python3_cpu() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization @@ -550,9 +546,7 @@ unittest_ubuntu_python3_cpu() { unittest_ubuntu_python3_cpu_mkldnn() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl @@ -561,9 +555,7 @@ unittest_ubuntu_python3_cpu_mkldnn() { unittest_ubuntu_python2_gpu() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -595,9 +587,7 @@ tutorialtest_ubuntu_python2_gpu() { unittest_ubuntu_python3_gpu() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -615,9 +605,7 @@ unittest_ubuntu_python3_gpu_nocudnn() { unittest_ubuntu_python2_quantization_gpu() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu } @@ -627,9 +615,7 @@ unittest_ubuntu_python2_quantization_gpu() { unittest_ubuntu_python3_quantization_gpu() { set -ex export PYTHONPATH=./python/ - # MXNET_MKLDNN_DEBUG is buggy and produces false positives - # https://github.com/apache/incubator-mxnet/issues/10026 - #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present + export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu } From d0a68c6fd0da1551e588bae6f1b4140839d19edf Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Thu, 9 Aug 2018 11:20:51 -0700 Subject: [PATCH 6/9] add exclude debug flag --- src/operator/nn/lrn.cc | 2 ++ src/operator/nn/mkldnn/mkldnn_base.cc | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/operator/nn/lrn.cc b/src/operator/nn/lrn.cc index 6b3d7c818378..30a752340a5b 100644 --- a/src/operator/nn/lrn.cc +++ b/src/operator/nn/lrn.cc @@ -204,6 +204,8 @@ NNVM_REGISTER_OP(_backward_LRN) .set_attr("TIsBackward", true) #if MXNET_USE_MKLDNN == 1 .set_attr("FComputeEx", LRNGradComputeExCPU) +// Native compute requires norm while MKLDNN does not so cannot be compared in debug mode +.set_attr("TExcludeMKLDNNDebug", true) #endif .set_attr("FCompute", LRNGradCompute); diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc index 8ad42a46c1a0..677e1e67b708 100644 --- a/src/operator/nn/mkldnn/mkldnn_base.cc +++ b/src/operator/nn/mkldnn/mkldnn_base.cc @@ -496,6 +496,11 @@ void OpCheck::Run(mxnet::FCompute fn, const nnvm::NodeAttrs &attrs, const std::vector &inputs_, const std::vector &req, const std::vector &outputs_) { + + static auto& is_excluded = Op::GetAttr("TExcludeMKLDNNDebug"); + + if (is_excluded.get(attrs.op, false)) return; + std::vector in_blobs(inputs.size()); for (size_t i = 0; i < in_blobs.size(); i++) in_blobs[i] = inputs[i].data(); std::vector out_blobs(outputs.size()); From eb04161c927464e6beb830f62e84a07eaa6fe0c5 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Thu, 9 Aug 2018 11:28:30 -0700 Subject: [PATCH 7/9] fix lint --- src/operator/nn/mkldnn/mkldnn_base.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc index 677e1e67b708..2edbe22486e4 100644 --- a/src/operator/nn/mkldnn/mkldnn_base.cc +++ b/src/operator/nn/mkldnn/mkldnn_base.cc @@ -496,11 +496,8 @@ void OpCheck::Run(mxnet::FCompute fn, const nnvm::NodeAttrs &attrs, const std::vector &inputs_, const std::vector &req, const std::vector &outputs_) { - static auto& is_excluded = Op::GetAttr("TExcludeMKLDNNDebug"); - if (is_excluded.get(attrs.op, false)) return; - std::vector in_blobs(inputs.size()); for (size_t i = 0; i < in_blobs.size(); i++) in_blobs[i] = inputs[i].data(); std::vector out_blobs(outputs.size()); From 4aaf82f63c764435c103dc62b89760b0e6a107c4 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Sun, 12 Aug 2018 15:29:53 -0700 Subject: [PATCH 8/9] add warning log for excluded op --- src/operator/nn/mkldnn/mkldnn_base.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/operator/nn/mkldnn/mkldnn_base.cc b/src/operator/nn/mkldnn/mkldnn_base.cc index 2edbe22486e4..27c574deae53 100644 --- a/src/operator/nn/mkldnn/mkldnn_base.cc +++ b/src/operator/nn/mkldnn/mkldnn_base.cc @@ -497,7 +497,10 @@ void OpCheck::Run(mxnet::FCompute fn, const nnvm::NodeAttrs &attrs, const std::vector &req, const std::vector &outputs_) { static auto& is_excluded = Op::GetAttr("TExcludeMKLDNNDebug"); - if (is_excluded.get(attrs.op, false)) return; + if (is_excluded.get(attrs.op, false)) { + LOG(WARNING) << attrs.op->name << " not checked. TExcludeMKLDNNDebug flag present"; + return; + } std::vector in_blobs(inputs.size()); for (size_t i = 0; i < in_blobs.size(); i++) in_blobs[i] = inputs[i].data(); std::vector out_blobs(outputs.size()); From 3e635d7041ae3787800c71e6326c1d7104189443 Mon Sep 17 00:00:00 2001 From: Alexander Zai Date: Mon, 13 Aug 2018 10:47:43 -0700 Subject: [PATCH 9/9] retrigger