Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handled unhandled exception to show bug report message #4565

Merged
merged 22 commits into from
Jan 17, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions samcli/commands/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
"""
Class containing error conditions that are exposed to the user.
"""
import urllib.parse
hawflau marked this conversation as resolved.
Show resolved Hide resolved
import typing as t
hawflau marked this conversation as resolved.
Show resolved Hide resolved
import traceback

import click

Expand All @@ -25,6 +28,41 @@ def __init__(self, message, wrapped_from=None):
click.ClickException.__init__(self, message)


class UnhandledException(click.ClickException):
"""
Exception class to re-wrap any exception that is not a UserException.
Typically this means there is a bug in SAM CLI.
"""

GH_BUG_REPORT_URL = "https://github.com/aws/aws-sam-cli/issues/new?template=Bug_report.md&title={title}"
hawflau marked this conversation as resolved.
Show resolved Hide resolved
exit_code = 255

def __init__(self, command: str, exception: Exception) -> None:
self._command = command
self._exception = exception
self.__traceback__ = self._exception.__traceback__

click.ClickException.__init__(self, type(exception).__name__)

def show(self, file: t.Optional[t.IO] = None) -> None:
hawflau marked this conversation as resolved.
Show resolved Hide resolved
hawflau marked this conversation as resolved.
Show resolved Hide resolved
"""Overriding show to customize printing stack trace and message"""
if file is None:
file = click._compat.get_text_stderr() # pylint: disable=protected-access

tb = "".join(traceback.format_tb(self.__traceback__))
click.echo(f"\nTraceback:\n{tb}", file=file, err=True)

url = self.GH_BUG_REPORT_URL.format(
title=urllib.parse.quote(f"{self._command} - {type(self._exception).__name__}")
)
msg = (
f'While trying to execute "{self._command}", we encountered an unexpected error.\n'
hawflau marked this conversation as resolved.
Show resolved Hide resolved
"To create a bug report, follow the Github issue template below:\n"
f"{url}"
)
click.secho(msg, file=file, err=True, fg="yellow")


class CredentialsError(UserException):
"""
Exception class when credentials that have been passed are invalid.
Expand Down
16 changes: 9 additions & 7 deletions samcli/lib/telemetry/metric.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from samcli.cli.context import Context
from samcli.cli.global_config import GlobalConfig
from samcli.commands._utils.experimental import get_all_experimental_statues
from samcli.commands.exceptions import UserException
from samcli.commands.exceptions import UserException, UnhandledException
from samcli.lib.hook.hook_config import HookPackageConfig
from samcli.lib.hook.hook_wrapper import INTERNAL_PACKAGES_ROOT
from samcli.lib.hook.exceptions import InvalidHookPackageConfigException
Expand Down Expand Up @@ -148,16 +148,18 @@ def wrapped(*args, **kwargs):
# Execute the function and capture return value. This is returned back by the wrapper
# First argument of all commands should be the Context
return_value = func(*args, **kwargs)
except UserException as ex:
except (UserException, click.BadOptionUsage, click.BadArgumentUsage, click.BadParameter) as ex:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why did we add more exceptions to catch?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I understand now, but why did we add only these? Should we just catch all click exceptions? Or at least a few others that are here: https://click.palletsprojects.com/en/8.1.x/api/#exceptions

Copy link
Contributor Author

@hawflau hawflau Jan 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. I missed click.UsageError.

The idea of grouping these click exceptions together is that these exceptions should be fixable by users. Any user-fixable-exception that is thrown within the call stacks wrapped by @track_command should be caught by this line. I don't want to catch click.ClickException in general because that could be over-catching. I'd rather an exception to be an Unhandled Exception by default.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha. What about NoSuchOption?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see any reason we should use NoSuchOption in our own code. So don't think we should include it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe I'm misunderstanding but wouldn't NoSuchOption be raised by click when a user provides an option that doesn't exist? Or are you saying that wouldn't even reach this code anyway?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It won't reach this code. We should have all @option declared before @track_command

# Capture exception information and re-raise it later so we can first send metrics.
exception = ex
exit_code = ex.exit_code
if ex.wrapped_from is None:
exit_reason = type(ex).__name__
else:
# TODO (hawflau): review exitcode meaning. We now understand exitcode 1 as errors fixable by users.
exit_code = 1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did this change from ex.exit_code to 1?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

click.BadOptionUsage and other click exception caught here come with exitcode=2 by default. In our Telemetry, they were, however, classified as 255 (as we didn't catch any them). Today, we only have 1 and 255 exitcodes in our Telemetry. We should revisit the exitcode meaning someday. But for now, re-classifying them as exitcode 1 seems more correct to me.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But why not use whatever actual exit code is thrown, instead of only either 1 or 255?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now I don't want to introduce new exitcode (e.g. 2) into Telemetry, until we have clearer understandings about how we want to use the exitcode. Today, click.BadOptionUsage and click.UsageError (maybe a couple other user-fixable exceptions as well) go into Telemetry with exitcode 255, which is certainly incorrect. I just want to fix this first.

if hasattr(ex, "wrapped_from") and ex.wrapped_from:
exit_reason = ex.wrapped_from
else:
exit_reason = type(ex).__name__
except Exception as ex:
exception = ex
command = ctx.command_path if ctx else ""
torresxb1 marked this conversation as resolved.
Show resolved Hide resolved
exception = UnhandledException(command, ex)
# Standard Unix practice to return exit code 255 on fatal/unhandled exit.
exit_code = 255
exit_reason = type(ex).__name__
Expand Down
18 changes: 9 additions & 9 deletions tests/unit/lib/telemetry/test_metric.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
import platform
import time

from parameterized import parameterized

import samcli

from unittest import TestCase
from unittest.mock import patch, Mock, ANY, call

import pytest
from parameterized import parameterized

import samcli
from samcli.lib.hook.exceptions import InvalidHookPackageConfigException, InvalidHookWrapperException
from samcli.lib.iac.plugins_interfaces import ProjectTypes
from samcli.lib.telemetry.event import EventTracker
Expand All @@ -28,7 +26,7 @@
ProjectDetails,
_send_command_run_metrics,
)
from samcli.commands.exceptions import UserException
from samcli.commands.exceptions import UserException, UnhandledException


class TestSendInstalledMetric(TestCase):
Expand Down Expand Up @@ -194,7 +192,8 @@ def test_records_exceptions(self, exception, expected_exception, expected_code,
def real_fn():
raise exception

with self.assertRaises(exception.__class__) as context:
expected_exception_class = exception.__class__ if isinstance(exception, UserException) else UnhandledException
with self.assertRaises(expected_exception_class) as context:
track_command(real_fn)()
self.assertEqual(
context.exception,
Expand Down Expand Up @@ -261,17 +260,18 @@ def func(**kwargs):
)
@patch("samcli.lib.telemetry.metric.Context")
@patch("samcli.lib.telemetry.metric._send_command_run_metrics")
def test_context_contains_exception(self, expected_exception, expected_code, run_metrics_mock, context_mock):
self.context_mock.exception = expected_exception("some exception")
def test_context_contains_exception(self, exception, exitcode, run_metrics_mock, context_mock):
self.context_mock.exception = exception("some exception")
context_mock.get_current_context.return_value = self.context_mock

mocked_func = Mock()

expected_exception = exception if exitcode == 1 else UnhandledException
with self.assertRaises(expected_exception):
track_command(mocked_func)()

mocked_func.assert_not_called()
run_metrics_mock.assert_called_with(self.context_mock, ANY, expected_exception.__name__, expected_code)
run_metrics_mock.assert_called_with(self.context_mock, ANY, exception.__name__, exitcode)


class TestSendCommandMetrics(TestCase):
Expand Down