Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed auto codegen for intermediate tensors #39797

Merged
merged 17 commits into from
Mar 1, 2022
Merged

Fixed auto codegen for intermediate tensors #39797

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
183b80f
Refactored GradNodeAccumulation data structure and behaviour
jim19930609 Feb 14, 2022
aab36af
Fixed CI issues
jim19930609 Feb 15, 2022
27cebd7
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
jim19930609 Feb 16, 2022
48ee099
Fix compilation issues
jim19930609 Feb 16, 2022
200806d
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
jim19930609 Feb 19, 2022
9f04b85
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
jim19930609 Feb 21, 2022
d342316
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
jim19930609 Feb 21, 2022
eda1afb
Fixed minor issues
jim19930609 Feb 21, 2022
2d21df3
Reverted changes for intermediate and OverwriteOutput
jim19930609 Feb 22, 2022
5333641
fixed minor issue
jim19930609 Feb 22, 2022
62ca145
Fixed auto codegen for intermediate tensors
jim19930609 Feb 22, 2022
99988c6
Removed restriction on AccumulationNode modification
jim19930609 Feb 23, 2022
f19a052
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
jim19930609 Feb 24, 2022
70fd4b8
Fixed CI Coverage issues
jim19930609 Feb 25, 2022
c975e47
Adjusted Log contents
jim19930609 Feb 28, 2022
afbef76
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
jim19930609 Feb 28, 2022
3099697
Fixed CI issues
jim19930609 Mar 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions paddle/fluid/eager/auto_code_generator/eager_generator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1156,11 +1156,13 @@ static std::string GenerateGradNodeCreationContent(
grad_node_creation_str += paddle::string::Sprintf(
SET_OUT_RANK_TEMPLATE, output_autograd_name, output_position);

const char* SET_HISTORY_TEMPLATE =
" egr::EagerUtils::SetHistory(&%s, grad_node);\n";
grad_node_creation_str +=
paddle::string::Sprintf(SET_HISTORY_TEMPLATE, output_autograd_name);

// Intermediate Tensor does not require SetHistory
if (!output.intermediate()) {
const char* SET_HISTORY_TEMPLATE =
" egr::EagerUtils::SetHistory(&%s, grad_node);\n";
grad_node_creation_str +=
paddle::string::Sprintf(SET_HISTORY_TEMPLATE, output_autograd_name);
}
const char* SET_GRAD_IN_META_TEMPLATE =
" grad_node->SetGradInMeta(&%s, %d);\n";
grad_node_creation_str += paddle::string::Sprintf(
Expand All @@ -1173,17 +1175,20 @@ static std::string GenerateGradNodeCreationContent(
grad_node_creation_str += paddle::string::Sprintf(
SET_OUT_RANK_TEMPLATE, output_autograd_name, output_position);

const char* SET_HISTORY_TEMPLATE =
" egr::EagerUtils::SetHistory(%s, grad_node);\n";
grad_node_creation_str +=
paddle::string::Sprintf(SET_HISTORY_TEMPLATE, output_autograd_name);

// Intermediate Tensor does not require SetHistory
if (!output.intermediate()) {
const char* SET_HISTORY_TEMPLATE =
" egr::EagerUtils::SetHistory(%s, grad_node);\n";
grad_node_creation_str +=
paddle::string::Sprintf(SET_HISTORY_TEMPLATE, output_autograd_name);
}
const char* SET_GRAD_IN_META_TEMPLATE =
" grad_node->SetGradInMeta(%s, %d);\n";
grad_node_creation_str += paddle::string::Sprintf(
SET_GRAD_IN_META_TEMPLATE, output_autograd_name, output_position);
}

// Intermediate Tensor does not require CheckAndRetainGrad
if (!output.intermediate()) {
VLOG(6) << "Generated Call RetainGradForTensor";
const char* RETAIN_GRAD_TEMPLATE =
Expand Down
7 changes: 4 additions & 3 deletions paddle/fluid/eager/backward.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,10 +221,11 @@ void RunBackward(const std::vector<paddle::experimental::Tensor>& tensors,
<< " 's name is: " << grad_output_tensor.name();

auto* next_node = next_node_shared.get();

if (!node_input_buffers_dict.count(next_node)) {
node_input_buffers_dict[next_node] =
std::make_unique<GradTensorHolder>(next_node->InputMeta());
const auto& input_meta = next_node->InputMeta();
auto grad_tensor_holder =
std::make_unique<GradTensorHolder>(input_meta);
node_input_buffers_dict[next_node] = std::move(grad_tensor_holder);
}
VLOG(6) << "Sum grad inputs for edge slot: " << edge_rank.first
<< ", rank: " << edge_rank.second;
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/eager/grad_node_info.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ GradNodeBase::ApplyGradientHooks(
VLOG(8) << "Run Hook for tensor: " << tensors[slot_id][rank].name();
out = hook(tensors[slot_id][rank]);
} else {
// If more than one hook is registered, the input to the next hook func
// If more than one hook is registered, the input to the next hook func
// should be the output of the previous hook
out = hook(out);
}
Expand Down
17 changes: 17 additions & 0 deletions paddle/fluid/eager/utils.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,12 +122,29 @@ paddle::experimental::Tensor* EagerUtils::mutable_grad(
void EagerUtils::SetHistory(std::vector<AutogradMeta*>* autograd_metas,
const std::shared_ptr<GradNodeBase>& grad_node) {
for (const auto& autograd_meta : *autograd_metas) {
if (dynamic_cast<GradNodeAccumulation*>(autograd_meta->GradNode())) {
VLOG(6)
<< "Warning: Reseting GradNodeAccumulation for leaf tensor is not "
"allowed."
"You're not expected to hit this error, likely an internal issue "
"with Paddle."
"Please contact Paddle developers for more details.";
}
autograd_meta->SetGradNode(grad_node);
}
}

void EagerUtils::SetHistory(AutogradMeta* autograd_meta,
const std::shared_ptr<GradNodeBase>& grad_node) {
if (dynamic_cast<GradNodeAccumulation*>(autograd_meta->GradNode())) {
VLOG(6)
<< "Warning: Reseting GradNodeAccumulation for leaf tensor is not "
"allowed."
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bad log

"You're not expected to hit this error, likely an internal issue "
"with Paddle."
"Please contact Paddle developers for more details.";
}

autograd_meta->SetGradNode(grad_node);
}

Expand Down