From c8e988f6f0b8128ce241845112fa444478c1958f Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Tue, 23 Nov 2021 10:23:06 +0000 Subject: [PATCH 1/3] Added EagerUtils to Eager Dygraph --- paddle/fluid/eager/CMakeLists.txt | 3 + paddle/fluid/eager/api/CMakeLists.txt | 1 + paddle/fluid/eager/api/utils/CMakeLists.txt | 1 + paddle/fluid/eager/api/utils/global_utils.cc | 22 ++++ paddle/fluid/eager/api/utils/global_utils.h | 62 +++++++++++ paddle/fluid/eager/tests/CMakeLists.txt | 3 +- .../eager/tests/task_tests/CMakeLists.txt | 1 + .../tests/task_tests/eager_utils_test.cc | 101 +++++++++++++++++ paddle/fluid/eager/utils.cc | 104 ++++++++++++++++++ paddle/fluid/eager/utils.h | 68 ++++++++++++ 10 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 paddle/fluid/eager/api/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/utils/CMakeLists.txt create mode 100644 paddle/fluid/eager/api/utils/global_utils.cc create mode 100644 paddle/fluid/eager/api/utils/global_utils.h create mode 100644 paddle/fluid/eager/tests/task_tests/CMakeLists.txt create mode 100644 paddle/fluid/eager/tests/task_tests/eager_utils_test.cc create mode 100644 paddle/fluid/eager/utils.cc create mode 100644 paddle/fluid/eager/utils.h diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index a79b451b54431b..9d74aae4874afb 100644 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -1,3 +1,6 @@ +add_subdirectory(api) add_subdirectory(tests) cc_library(grad_node_info SRCS grad_node_info.cc DEPS pten pten_api) cc_library(autograd_meta SRCS autograd_meta.cc DEPS pten pten_api) + +cc_library(utils SRCS utils.cc DEPS pten pten_api global_utils layer proto_desc operator op_registry variable_helper memcpy scale_op autograd_meta) diff --git a/paddle/fluid/eager/api/CMakeLists.txt b/paddle/fluid/eager/api/CMakeLists.txt new file mode 100644 index 00000000000000..512d2b1553c8c9 --- /dev/null +++ b/paddle/fluid/eager/api/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(utils) diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt new file mode 100644 index 00000000000000..5168f1fc02489c --- /dev/null +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -0,0 +1 @@ +cc_library(global_utils SRCS global_utils.cc DEPS enforce) diff --git a/paddle/fluid/eager/api/utils/global_utils.cc b/paddle/fluid/eager/api/utils/global_utils.cc new file mode 100644 index 00000000000000..3a6a05eb1bfc8b --- /dev/null +++ b/paddle/fluid/eager/api/utils/global_utils.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "paddle/fluid/eager/api/utils/global_utils.h" + +namespace egr { + +Controller* Controller::controller_ = new Controller(); + +} // namespace egr diff --git a/paddle/fluid/eager/api/utils/global_utils.h b/paddle/fluid/eager/api/utils/global_utils.h new file mode 100644 index 00000000000000..16e7ef8a58e666 --- /dev/null +++ b/paddle/fluid/eager/api/utils/global_utils.h @@ -0,0 +1,62 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/platform/enforce.h" + +namespace egr { + +class UniqueNameGenerator { + public: + explicit UniqueNameGenerator(std::string prefix = "") : prefix_(prefix) {} + std::string Generate(std::string key = "eager_tmp") { + return prefix_ + key + "_" + std::to_string(id_++); + } + + private: + std::atomic id_{0}; + std::string prefix_; +}; + +// Global +class Controller { + public: + static Controller& Instance() { return *controller_; } + const paddle::platform::Place& GetExpectedPlace() const { + return *expected_place_.get(); + } + void SetExpectedPlace(const paddle::platform::Place& place) { + expected_place_ = std::make_shared(place); + } + void SetAMPLevel(int level) { amp_level_ = level; } + int GetAMPLevel() const { return amp_level_; } + bool HasGrad() const { return has_grad_; } + std::string GenerateUniqueName(std::string key = "eager_tmp") { + return generator_->Generate(key); + } + + private: + Controller() = default; + static Controller* controller_; + std::shared_ptr expected_place_ = nullptr; + int amp_level_ = 0; + bool has_grad_ = true; + std::unique_ptr generator_{new UniqueNameGenerator()}; + DISABLE_COPY_AND_ASSIGN(Controller); +}; + +} // namespace egr diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt index 572740e03c66c1..81c18886675d75 100644 --- a/paddle/fluid/eager/tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -1,2 +1,3 @@ -set(eager_deps pten pten_api) +set(eager_deps pten pten_api pten_tensor utils) add_subdirectory(data_structure_tests) +add_subdirectory(task_tests) diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt new file mode 100644 index 00000000000000..701deacc153352 --- /dev/null +++ b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt @@ -0,0 +1 @@ +cc_test(test_egr_ds_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps}) diff --git a/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc b/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc new file mode 100644 index 00000000000000..566a7836dff933 --- /dev/null +++ b/paddle/fluid/eager/tests/task_tests/eager_utils_test.cc @@ -0,0 +1,101 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "gtest/gtest.h" + +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/eager/grad_node_info.h" +#include "paddle/fluid/eager/tests/data_structure_tests/grad_node_test.h" +#include "paddle/fluid/eager/utils.h" +#include "paddle/pten/api/lib/utils/allocator.h" + +#include "paddle/pten/core/kernel_registry.h" + +// TODO(jiabin): remove nolint here!!! +using namespace egr; // NOLINT + +TEST(EagerUtils, AutoGradMeta) { + // Construct Eager Tensor + pten::DenseTensorMeta meta = pten::DenseTensorMeta( + pten::DataType::FLOAT32, paddle::framework::make_ddim({1, 1})); + std::shared_ptr dt0 = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta); + dt0->mutable_data()[0] = 10.0; + EagerTensor et0 = EagerTensor(dt0); + + std::shared_ptr dt1 = std::make_shared( + std::make_shared( + paddle::platform::CPUPlace()), + meta); + dt1->mutable_data()[0] = 20.0; + EagerTensor et1 = EagerTensor(dt1); + + std::vector ets = {et0, et1}; + auto test_node = std::make_shared(); + + // unsafe_autograd_meta() + // autograd_meta() + // multi_autograd_meta() + AutogradMeta* autograd_meta0 = EagerUtils::autograd_meta(&et0); + AutogradMeta* autograd_meta1 = EagerUtils::autograd_meta(&et1); + + AutogradMeta* unsafe_autograd_meta_after = + EagerUtils::unsafe_autograd_meta(et0); + CHECK_NOTNULL(unsafe_autograd_meta_after); + + std::vector autograd_metas = + EagerUtils::multi_autograd_meta(&ets); + std::vector unsafe_autograd_metas = + EagerUtils::unsafe_autograd_meta(&ets); + CHECK_NOTNULL(unsafe_autograd_metas[0]); + CHECK_NOTNULL(unsafe_autograd_metas[1]); + + // Set Autograd Meta + autograd_meta0->SetSingleOutRankWithSlot(0, 1); + + autograd_meta0->SetGradNode(test_node); + + // OutRankInfo() + std::pair out_rank_info0 = EagerUtils::OutRankInfo(et0); + CHECK_EQ(static_cast(out_rank_info0.first), 0); + CHECK_EQ(static_cast(out_rank_info0.second), 1); + + // grad_node() + std::shared_ptr grad_node0 = EagerUtils::grad_node(et0); + CHECK_NOTNULL(grad_node0.get()); + + EagerUtils::SetHistory(autograd_meta1, test_node); + EagerUtils::SetHistory({autograd_meta1}, test_node); + std::shared_ptr grad_node1 = EagerUtils::grad_node(et1); + CHECK_NOTNULL(grad_node1.get()); + + // SetOutRankWithSlot() + EagerUtils::SetOutRankWithSlot(autograd_meta1, 0); + std::pair out_rank_info1 = EagerUtils::OutRankInfo(et1); + CHECK_EQ(static_cast(out_rank_info1.first), 0); + CHECK_EQ(static_cast(out_rank_info1.second), 0); + + EagerUtils::SetOutRankWithSlot(&autograd_metas, 0); + std::pair out_rank_info2 = EagerUtils::OutRankInfo(et0); + CHECK_EQ(static_cast(out_rank_info2.first), 0); + CHECK_EQ(static_cast(out_rank_info2.second), 0); + + std::pair out_rank_info3 = EagerUtils::OutRankInfo(et1); + CHECK_EQ(static_cast(out_rank_info3.first), 0); + CHECK_EQ(static_cast(out_rank_info3.second), 1); +} diff --git a/paddle/fluid/eager/utils.cc b/paddle/fluid/eager/utils.cc new file mode 100644 index 00000000000000..b2cc8c94fc7d34 --- /dev/null +++ b/paddle/fluid/eager/utils.cc @@ -0,0 +1,104 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/eager/utils.h" +#include "paddle/fluid/eager/api/utils/global_utils.h" + +#include "paddle/pten/api/all.h" +#include "paddle/pten/common/layout.h" +#include "paddle/pten/core/tensor_meta.h" + +#include "paddle/fluid/framework/data_layout.h" +#include "paddle/fluid/framework/pten_utils.h" +#include "paddle/fluid/framework/variable.h" + +namespace egr { +/** + * Implementation of Eager Utils. +**/ + +AutogradMeta* EagerUtils::autograd_meta(egr::EagerTensor* target) { + auto* p_autograd_meta = target->get_autograd_meta(); + if (!p_autograd_meta) { + auto p_autograd_meta_ptr = std::make_shared(); + p_autograd_meta = p_autograd_meta_ptr.get(); + target->set_autograd_meta(p_autograd_meta_ptr); + } + return static_cast(p_autograd_meta); +} + +AutogradMeta* EagerUtils::unsafe_autograd_meta(const egr::EagerTensor& target) { + auto* p_autograd_meta = target.get_autograd_meta(); + PADDLE_ENFORCE(p_autograd_meta, + paddle::platform::errors::Fatal( + "Null autograd_meta gotten from unsafe_autograd_meta()")); + return static_cast(p_autograd_meta); +} + +std::vector EagerUtils::unsafe_autograd_meta( + std::vector* targets) { + std::vector metas; + for (const egr::EagerTensor& t : *targets) { + metas.push_back(unsafe_autograd_meta(t)); + } + return metas; +} + +std::vector EagerUtils::multi_autograd_meta( + std::vector* targets) { + std::vector ret; + ret.reserve(targets->size()); + + // for multi_autograd_meta we can tolerent it has nullptr. + for (auto& t : (*targets)) { + auto* p_autograd_meta = autograd_meta(&t); + ret.push_back(static_cast(p_autograd_meta)); + } + return ret; +} + +std::pair EagerUtils::OutRankInfo( + const egr::EagerTensor& target) { + return unsafe_autograd_meta(target)->OutRankInfo(); +} + +std::shared_ptr EagerUtils::grad_node( + const egr::EagerTensor& target) { + return unsafe_autograd_meta(target)->GetMutableGradNode(); +} + +void EagerUtils::SetHistory(std::vector* autograd_metas, + const std::shared_ptr& grad_node) { + for (const auto& autograd_meta : *autograd_metas) { + autograd_meta->SetGradNode(grad_node); + } +} + +void EagerUtils::SetHistory(AutogradMeta* autograd_meta, + const std::shared_ptr& grad_node) { + autograd_meta->SetGradNode(grad_node); +} + +void EagerUtils::SetOutRankWithSlot(std::vector* targets, + size_t slot_id) { + // Set OutRankInfo from 0 to size of targets + for (size_t i = 0; i < targets->size(); i++) { + (*targets)[i]->SetSingleOutRankWithSlot(slot_id, i); + } +} +void EagerUtils::SetOutRankWithSlot(AutogradMeta* target, size_t slot_id) { + target->SetSingleOutRankWithSlot(slot_id, 0); +} + +} // namespace egr diff --git a/paddle/fluid/eager/utils.h b/paddle/fluid/eager/utils.h new file mode 100644 index 00000000000000..4c291f2437f30e --- /dev/null +++ b/paddle/fluid/eager/utils.h @@ -0,0 +1,68 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/eager/autograd_meta.h" +#include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/fluid/eager/grad_node_info.h" + +#include "paddle/pten/api/all.h" + +namespace egr { + +/** + * EagerUtils is utils used to do some static conversion or autograd + * members access, this class is desinged to be a full static functional + * utils class + * **/ + +class EagerUtils { + public: + /** + * We have to use autograd_meta and multi_autograd_meta to initialize + * autograd_meta for tensor, since we can't init it in + * egr::EagerTensor's + * constructor (it's abstract class there) + * + * **/ + static AutogradMeta* autograd_meta(egr::EagerTensor* target); + + static std::vector multi_autograd_meta( + std::vector* targets); + + static std::pair OutRankInfo(const egr::EagerTensor& target); + + static std::shared_ptr grad_node( + const egr::EagerTensor& target); + + // Set history is used to set backward info during forward process, it will + // set forward var's autograd meta's grad node as current backward node. + static void SetHistory(std::vector* autograd_metas, + const std::shared_ptr& grad_node); + static void SetHistory(AutogradMeta* autograd_meta, + const std::shared_ptr& grad_node); + + // This is used for Set vector of tensors' rank + static void SetOutRankWithSlot(std::vector* targets, + size_t slot_id); + static void SetOutRankWithSlot(AutogradMeta* target, size_t slot_id); + + // This method will return an AutogradMeta pointer unsafely. + static AutogradMeta* unsafe_autograd_meta(const egr::EagerTensor& target); + static std::vector unsafe_autograd_meta( + std::vector* targets); +}; + +} // namespace egr From 2ba95805a3c37d77d070c1d6b9a8b94af828354b Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Wed, 24 Nov 2021 01:35:47 +0000 Subject: [PATCH 2/3] Purified include dependencies for global_utils --- paddle/fluid/eager/api/utils/CMakeLists.txt | 2 +- paddle/fluid/eager/api/utils/global_utils.h | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt index 5168f1fc02489c..2bce8e0bc30725 100644 --- a/paddle/fluid/eager/api/utils/CMakeLists.txt +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -1 +1 @@ -cc_library(global_utils SRCS global_utils.cc DEPS enforce) +cc_library(global_utils SRCS global_utils.cc DEPS place) diff --git a/paddle/fluid/eager/api/utils/global_utils.h b/paddle/fluid/eager/api/utils/global_utils.h index 16e7ef8a58e666..39db269a02649f 100644 --- a/paddle/fluid/eager/api/utils/global_utils.h +++ b/paddle/fluid/eager/api/utils/global_utils.h @@ -15,8 +15,9 @@ #pragma once -#include "paddle/fluid/eager/eager_tensor.h" -#include "paddle/fluid/platform/enforce.h" +#include +#include +#include "paddle/fluid/platform/place.h" namespace egr { From efcc61d836e0929ab1d43b21c054a74f36023c77 Mon Sep 17 00:00:00 2001 From: jim19930609 Date: Wed, 24 Nov 2021 03:20:04 +0000 Subject: [PATCH 3/3] Fixed merge conflicts --- paddle/fluid/eager/tests/task_tests/CMakeLists.txt | 2 +- paddle/fluid/eager/utils.cc | 14 -------------- paddle/fluid/eager/utils.h | 3 --- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt index 701deacc153352..6ab5c70b44e82e 100644 --- a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt @@ -1 +1 @@ -cc_test(test_egr_ds_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps}) +cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps}) diff --git a/paddle/fluid/eager/utils.cc b/paddle/fluid/eager/utils.cc index 06631a3a42a8bc..0b7e6e357dc14a 100644 --- a/paddle/fluid/eager/utils.cc +++ b/paddle/fluid/eager/utils.cc @@ -188,18 +188,4 @@ egr::EagerTensor EagerUtils::GetOutput( return EagerTensor((*(out.get()))); } -AutogradMeta* EagerUtils::unsafe_autograd_meta(const egr::EagerTensor& target) { - auto* p_autograd_meta = target.get_autograd_meta(); - PADDLE_ENFORCE(p_autograd_meta, - paddle::platform::errors::Fatal( - "Null autograd_meta gotten from unsafe_autograd_meta(), " - "if you are using unsafe_autograd_meta, please make sure " - "your tensor's autograd_meta is set")); - return static_cast(p_autograd_meta); -} - -std::pair EagerUtils::OutRankInfo( - const egr::EagerTensor& target) { - return unsafe_autograd_meta(target)->OutRankInfo(); -} } // namespace egr diff --git a/paddle/fluid/eager/utils.h b/paddle/fluid/eager/utils.h index c358b1de6b3a78..19692df78dd40f 100644 --- a/paddle/fluid/eager/utils.h +++ b/paddle/fluid/eager/utils.h @@ -130,9 +130,6 @@ class EagerUtils { iter.SetStopGradient(stop_gradient); iter.apply(std::forward(args)...); } - static std::pair OutRankInfo(const egr::EagerTensor& target); - // This method will return an AutogradMeta pointer unsafely. - static AutogradMeta* unsafe_autograd_meta(const egr::EagerTensor& target); // Intermidate needed remove this once we don't need legacy static std::vector> SyncToVars(