From 042a6b910b0bcda1ac35a00a717a9ff3804b357d Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 9 Jan 2023 16:35:36 -0500 Subject: [PATCH 1/7] Combine DbtProfileError log events --- core/dbt/events/proto_types.py | 81 +++++----------------------------- core/dbt/events/types.proto | 61 +++++-------------------- core/dbt/events/types.py | 54 ++++------------------- core/dbt/task/base.py | 19 +------- tests/unit/test_events.py | 5 --- 5 files changed, 32 insertions(+), 188 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index d972a98155e..f51b3b6bd96 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -200,7 +200,8 @@ class DbtProjectErrorExceptionMsg(betterproto.Message): class DbtProfileError(betterproto.Message): """A011""" - pass + exc: str = betterproto.string_field(1) + profiles: List[str] = betterproto.string_field(2) @dataclass @@ -209,71 +210,6 @@ class DbtProfileErrorMsg(betterproto.Message): data: "DbtProfileError" = betterproto.message_field(2) -@dataclass -class DbtProfileErrorException(betterproto.Message): - """A012""" - - exc: str = betterproto.string_field(1) - - -@dataclass -class DbtProfileErrorExceptionMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DbtProfileErrorException" = betterproto.message_field(2) - - -@dataclass -class ProfileListTitle(betterproto.Message): - """A013""" - - pass - - -@dataclass -class ProfileListTitleMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ProfileListTitle" = betterproto.message_field(2) - - -@dataclass -class ListSingleProfile(betterproto.Message): - """A014""" - - profile: str = betterproto.string_field(1) - - -@dataclass -class ListSingleProfileMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ListSingleProfile" = betterproto.message_field(2) - - -@dataclass -class NoDefinedProfiles(betterproto.Message): - """A015""" - - pass - - -@dataclass -class NoDefinedProfilesMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "NoDefinedProfiles" = betterproto.message_field(2) - - -@dataclass -class ProfileHelpMessage(betterproto.Message): - """A016""" - - pass - - -@dataclass -class ProfileHelpMessageMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ProfileHelpMessage" = betterproto.message_field(2) - - @dataclass class StarterProjectPath(betterproto.Message): """A017""" @@ -508,11 +444,16 @@ class ExposureNameDeprecationMsg(betterproto.Message): class FunctionDeprecated(betterproto.Message): """D008""" + function_name: str = betterproto.string_field(1) + reason: str = betterproto.string_field(2) + suggested_action: str = betterproto.string_field(3) + version: str = betterproto.string_field(4) + + +@dataclass +class FunctionDeprecatedMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - function_name: str = betterproto.string_field(2) - reason: str = betterproto.string_field(3) - suggested_action: str = betterproto.string_field(4) - version: str = betterproto.string_field(5) + data: "FunctionDeprecated" = betterproto.message_field(2) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index b8c7c42a01f..7b34b637c6f 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -152,6 +152,8 @@ message DbtProjectErrorExceptionMsg { // A011 message DbtProfileError { + string exc = 1; + repeated string profiles = 2; } message DbtProfileErrorMsg { @@ -159,52 +161,7 @@ message DbtProfileErrorMsg { DbtProfileError data = 2; } -// A012 -message DbtProfileErrorException { - string exc = 1; -} - -message DbtProfileErrorExceptionMsg { - EventInfo info = 1; - DbtProfileErrorException data = 2; -} - -// A013 -message ProfileListTitle { -} - -message ProfileListTitleMsg { - EventInfo info = 1; - ProfileListTitle data = 2; -} - -// A014 -message ListSingleProfile { - string profile = 1; -} - -message ListSingleProfileMsg { - EventInfo info = 1; - ListSingleProfile data = 2; -} - -// A015 -message NoDefinedProfiles { -} - -message NoDefinedProfilesMsg { - EventInfo info = 1; - NoDefinedProfiles data = 2; -} - -// A016 -message ProfileHelpMessage { -} - -message ProfileHelpMessageMsg { - EventInfo info = 1; - ProfileHelpMessage data = 2; -} +// Skipped A012, A013, A014, A015, A016 // A017 message StarterProjectPath { @@ -386,11 +343,15 @@ message ExposureNameDeprecationMsg { //D008 message FunctionDeprecated { + string function_name = 1; + string reason = 2; + string suggested_action = 3; + string version = 4; +} + +message FunctionDeprecatedMsg { EventInfo info = 1; - string function_name = 2; - string reason = 3; - string suggested_action = 4; - string version = 5; + FunctionDeprecated data = 2; } // E - DB Adapter diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index de8a9cf2c99..c255a501b09 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -140,56 +140,20 @@ def code(self): return "A011" def message(self) -> str: - return "Encountered an error while reading profiles:" - - -@dataclass -class DbtProfileErrorException(ErrorLevel, pt.DbtProfileErrorException): - def code(self): - return "A012" - - def message(self) -> str: - return f" ERROR: {str(self.exc)}" - - -@dataclass -class ProfileListTitle(InfoLevel, pt.ProfileListTitle): - def code(self): - return "A013" - - def message(self) -> str: - return "Defined profiles:" - - -@dataclass -class ListSingleProfile(InfoLevel, pt.ListSingleProfile): - def code(self): - return "A014" - - def message(self) -> str: - return f" - {self.profile}" - - -@dataclass -class NoDefinedProfiles(InfoLevel, pt.NoDefinedProfiles): - def code(self): - return "A015" - - def message(self) -> str: - return "There are no profiles defined in your profiles.yml file" - - -@dataclass -class ProfileHelpMessage(InfoLevel, pt.ProfileHelpMessage): - def code(self): - return "A016" + msg = "Encountered an error while reading profiles:\n" f" ERROR: {str(self.exc)}" + if self.profiles: + msg += "Defined profiles:\n" + for profile in self.profiles: + msg += f" - {profile}" + else: + msg += "There are no profiles defined in your profiles.yml file" - def message(self) -> str: - return """ + msg += """ For more information on configuring profiles, please consult the dbt docs: https://docs.getdbt.com/docs/configure-your-profile """ + return msg @dataclass diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index e448a15c1d2..633438a5b59 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -27,11 +27,6 @@ DbtProjectError, DbtProjectErrorException, DbtProfileError, - DbtProfileErrorException, - ProfileListTitle, - ListSingleProfile, - NoDefinedProfiles, - ProfileHelpMessage, CatchableExceptionOnRun, InternalExceptionOnRun, GenericExceptionOnRun, @@ -108,20 +103,8 @@ def from_args(cls, args): tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc except dbt.exceptions.DbtProfileError as exc: - fire_event(DbtProfileError()) - fire_event(DbtProfileErrorException(exc=str(exc))) - all_profiles = read_profiles(flags.PROFILES_DIR).keys() - - if len(all_profiles) > 0: - fire_event(ProfileListTitle()) - for profile in all_profiles: - fire_event(ListSingleProfile(profile=profile)) - else: - fire_event(NoDefinedProfiles()) - - fire_event(ProfileHelpMessage()) - + fire_event(DbtProfileError(exc=str(exc), profiles=all_profiles)) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc return cls(args, config) diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 17af8f94369..237783c7401 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -114,11 +114,6 @@ def test_event_codes(self): DbtProjectError(), DbtProjectErrorException(exc=""), DbtProfileError(), - DbtProfileErrorException(exc=""), - ProfileListTitle(), - ListSingleProfile(profile=""), - NoDefinedProfiles(), - ProfileHelpMessage(), StarterProjectPath(dir=""), ConfigFolderDirectory(dir=""), NoSampleProfileFound(adapter=""), From 11731cba070c9baefc252f87c823bf6182509a21 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 9 Jan 2023 17:02:12 -0500 Subject: [PATCH 2/7] Combine DbtProjectErrorException with DbtProjectError --- core/dbt/events/proto_types.py | 15 +-------------- core/dbt/events/types.proto | 11 ++--------- core/dbt/events/types.py | 13 +++++-------- core/dbt/task/base.py | 4 +--- tests/unit/test_events.py | 1 - 5 files changed, 9 insertions(+), 35 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index f51b3b6bd96..650a73cf740 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -174,7 +174,7 @@ class InvalidVarsYAMLMsg(betterproto.Message): class DbtProjectError(betterproto.Message): """A009""" - pass + exc: str = betterproto.string_field(1) @dataclass @@ -183,19 +183,6 @@ class DbtProjectErrorMsg(betterproto.Message): data: "DbtProjectError" = betterproto.message_field(2) -@dataclass -class DbtProjectErrorException(betterproto.Message): - """A010""" - - exc: str = betterproto.string_field(1) - - -@dataclass -class DbtProjectErrorExceptionMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DbtProjectErrorException" = betterproto.message_field(2) - - @dataclass class DbtProfileError(betterproto.Message): """A011""" diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 7b34b637c6f..2e6c0e742ab 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -133,6 +133,7 @@ message InvalidVarsYAMLMsg { // A009 message DbtProjectError { + string exc = 1; } message DbtProjectErrorMsg { @@ -140,15 +141,7 @@ message DbtProjectErrorMsg { DbtProjectError data = 2; } -// A010 -message DbtProjectErrorException { - string exc = 1; -} - -message DbtProjectErrorExceptionMsg { - EventInfo info = 1; - DbtProjectErrorException data = 2; -} +// Skipped A010 // A011 message DbtProfileError { diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index c255a501b09..1066ab6b5d3 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -122,16 +122,13 @@ def code(self): return "A009" def message(self) -> str: - return "Encountered an error while reading the project:" + msg = "Encountered an error while reading the project:" + if self.exc: + msg += f" ERROR: {str(self.exc)}" + return msg -@dataclass -class DbtProjectErrorException(ErrorLevel, pt.DbtProjectErrorException): - def code(self): - return "A010" - - def message(self) -> str: - return f" ERROR: {str(self.exc)}" +# Skipped A010 @dataclass diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 633438a5b59..0e4d93b688a 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -25,7 +25,6 @@ from dbt.events.functions import fire_event from dbt.events.types import ( DbtProjectError, - DbtProjectErrorException, DbtProfileError, CatchableExceptionOnRun, InternalExceptionOnRun, @@ -97,8 +96,7 @@ def from_args(cls, args): # for the clean or deps tasks config = cls.ConfigType.from_args(args) except dbt.exceptions.DbtProjectError as exc: - fire_event(DbtProjectError()) - fire_event(DbtProjectErrorException(exc=str(exc))) + fire_event(DbtProjectError(exc=str(exc))) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 237783c7401..112155ae42c 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -112,7 +112,6 @@ def test_event_codes(self): MissingProfileTarget(profile_name="", target_name=""), InvalidVarsYAML(), DbtProjectError(), - DbtProjectErrorException(exc=""), DbtProfileError(), StarterProjectPath(dir=""), ConfigFolderDirectory(dir=""), From 36bfe2daf89130aba3ecaf8f3061b625118a9823 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 9 Jan 2023 18:36:21 -0500 Subject: [PATCH 3/7] Combine cache logging events --- core/dbt/adapters/cache.py | 77 ++++++++------- core/dbt/events/proto_types.py | 173 +++------------------------------ core/dbt/events/types.proto | 133 +++---------------------- core/dbt/events/types.py | 142 +++++++-------------------- tests/unit/test_events.py | 31 +----- 5 files changed, 104 insertions(+), 452 deletions(-) diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 90c4cab27fb..430c79d3b3a 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -16,21 +16,7 @@ TruncatedModelNameCausedCollision, ) from dbt.events.functions import fire_event, fire_event_if -from dbt.events.types import ( - AddLink, - AddRelation, - DropCascade, - DropMissingRelation, - DropRelation, - DumpAfterAddGraph, - DumpAfterRenameSchema, - DumpBeforeAddGraph, - DumpBeforeRenameSchema, - RenameSchema, - TemporaryRelation, - UncachedRelation, - UpdateReference, -) +from dbt.events.types import CacheAction, CacheDumpGraph import dbt.flags as flags from dbt.utils import lowercase @@ -281,7 +267,7 @@ def _add_link(self, referenced_key, dependent_key): referenced.add_reference(dependent) - # TODO: Is this dead code? I can't seem to find it grepping the codebase. + # This is called in plugins/postgres/dbt/adapters/postgres/impl.py def add_link(self, referenced, dependent): """Add a link between two relations to the database. If either relation does not exist, it will be added as an "external" relation. @@ -303,9 +289,9 @@ def add_link(self, referenced, dependent): # referring to a table outside our control. There's no need to make # a link - we will never drop the referenced relation during a run. fire_event( - UncachedRelation( - dep_key=_make_msg_from_ref_key(dep_key), + CacheAction( ref_key=_make_msg_from_ref_key(ref_key), + ref_key_2=_make_msg_from_ref_key(dep_key), ) ) return @@ -318,8 +304,10 @@ def add_link(self, referenced, dependent): dependent = dependent.replace(type=referenced.External) self.add(dependent) fire_event( - AddLink( - dep_key=_make_msg_from_ref_key(dep_key), ref_key=_make_msg_from_ref_key(ref_key) + CacheAction( + action="add_link", + ref_key=_make_msg_from_ref_key(dep_key), + ref_key_2=_make_msg_from_ref_key(ref_key), ) ) with self.lock: @@ -332,12 +320,18 @@ def add(self, relation): :param BaseRelation relation: The underlying relation. """ cached = _CachedRelation(relation) - fire_event(AddRelation(relation=_make_ref_key_msg(cached))) - fire_event_if(flags.LOG_CACHE_EVENTS, lambda: DumpBeforeAddGraph(dump=self.dump_graph())) + fire_event_if( + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="before", action="adding", dump=self.dump_graph()), + ) + fire_event(CacheAction(action="add_relation", ref_key=_make_ref_key_msg(cached))) with self.lock: self._setdefault(cached) - fire_event_if(flags.LOG_CACHE_EVENTS, lambda: DumpAfterAddGraph(dump=self.dump_graph())) + fire_event_if( + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="after", action="adding", dump=self.dump_graph()), + ) def _remove_refs(self, keys): """Removes all references to all entries in keys. This does not @@ -365,16 +359,19 @@ def drop(self, relation): """ dropped_key = _make_ref_key(relation) dropped_key_msg = _make_ref_key_msg(relation) - fire_event(DropRelation(dropped=dropped_key_msg)) + fire_event(CacheAction(action="drop_relation", ref_key=dropped_key_msg)) with self.lock: if dropped_key not in self.relations: - fire_event(DropMissingRelation(relation=dropped_key_msg)) + fire_event(CacheAction(action="drop_missing_relation", ref_key=dropped_key_msg)) return consequences = self.relations[dropped_key].collect_consequences() # convert from a list of _ReferenceKeys to a list of ReferenceKeyMsgs consequence_msgs = [_make_msg_from_ref_key(key) for key in consequences] - - fire_event(DropCascade(dropped=dropped_key_msg, consequences=consequence_msgs)) + fire_event( + CacheAction( + action="drop_cascade", ref_key=dropped_key_msg, ref_list=consequence_msgs + ) + ) self._remove_refs(consequences) def _rename_relation(self, old_key, new_relation): @@ -397,12 +394,14 @@ def _rename_relation(self, old_key, new_relation): for cached in self.relations.values(): if cached.is_referenced_by(old_key): fire_event( - UpdateReference( - old_key=_make_ref_key_msg(old_key), - new_key=_make_ref_key_msg(new_key), - cached_key=_make_ref_key_msg(cached.key()), + CacheAction( + action="update_reference", + ref_key=_make_ref_key_msg(old_key), + ref_key_2=_make_ref_key_msg(new_key), + ref_key_3=_make_ref_key_msg(cached.key()), ) ) + cached.rename_key(old_key, new_key) self.relations[new_key] = relation @@ -430,7 +429,9 @@ def _check_rename_constraints(self, old_key, new_key): raise TruncatedModelNameCausedCollision(new_key, self.relations) if old_key not in self.relations: - fire_event(TemporaryRelation(key=_make_msg_from_ref_key(old_key))) + fire_event( + CacheAction(action="temporary_relation", ref_key=_make_msg_from_ref_key(old_key)) + ) return False return True @@ -449,13 +450,16 @@ def rename(self, old, new): old_key = _make_ref_key(old) new_key = _make_ref_key(new) fire_event( - RenameSchema( - old_key=_make_msg_from_ref_key(old_key), new_key=_make_msg_from_ref_key(new) + CacheAction( + action="rename_relation", + ref_key=_make_msg_from_ref_key(old_key), + ref_key_2=_make_msg_from_ref_key(new), ) ) fire_event_if( - flags.LOG_CACHE_EVENTS, lambda: DumpBeforeRenameSchema(dump=self.dump_graph()) + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="before", action="rename", dump=self.dump_graph()), ) with self.lock: @@ -465,7 +469,8 @@ def rename(self, old, new): self._setdefault(_CachedRelation(new)) fire_event_if( - flags.LOG_CACHE_EVENTS, lambda: DumpAfterRenameSchema(dump=self.dump_graph()) + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="after", action="rename", dump=self.dump_graph()), ) def get_relations(self, database: Optional[str], schema: Optional[str]) -> List[Any]: diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 650a73cf740..31089ad3fb4 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -750,186 +750,37 @@ class SchemaDropMsg(betterproto.Message): @dataclass -class UncachedRelation(betterproto.Message): +class CacheAction(betterproto.Message): """E022""" - dep_key: "ReferenceKeyMsg" = betterproto.message_field(1) + action: str = betterproto.string_field(1) ref_key: "ReferenceKeyMsg" = betterproto.message_field(2) + ref_key_2: "ReferenceKeyMsg" = betterproto.message_field(3) + ref_key_3: "ReferenceKeyMsg" = betterproto.message_field(4) + ref_list: List["ReferenceKeyMsg"] = betterproto.message_field(5) @dataclass -class UncachedRelationMsg(betterproto.Message): +class CacheActionMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "UncachedRelation" = betterproto.message_field(2) + data: "CacheAction" = betterproto.message_field(2) @dataclass -class AddLink(betterproto.Message): - """E023""" - - dep_key: "ReferenceKeyMsg" = betterproto.message_field(1) - ref_key: "ReferenceKeyMsg" = betterproto.message_field(2) - - -@dataclass -class AddLinkMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "AddLink" = betterproto.message_field(2) - - -@dataclass -class AddRelation(betterproto.Message): - """E024""" - - relation: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class AddRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "AddRelation" = betterproto.message_field(2) - - -@dataclass -class DropMissingRelation(betterproto.Message): - """E025""" - - relation: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class DropMissingRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DropMissingRelation" = betterproto.message_field(2) - - -@dataclass -class DropCascade(betterproto.Message): - """E026""" - - dropped: "ReferenceKeyMsg" = betterproto.message_field(1) - consequences: List["ReferenceKeyMsg"] = betterproto.message_field(2) - - -@dataclass -class DropCascadeMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DropCascade" = betterproto.message_field(2) - - -@dataclass -class DropRelation(betterproto.Message): - """E027""" - - dropped: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class DropRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DropRelation" = betterproto.message_field(2) - - -@dataclass -class UpdateReference(betterproto.Message): - """E028""" - - old_key: "ReferenceKeyMsg" = betterproto.message_field(1) - new_key: "ReferenceKeyMsg" = betterproto.message_field(2) - cached_key: "ReferenceKeyMsg" = betterproto.message_field(3) - - -@dataclass -class UpdateReferenceMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "UpdateReference" = betterproto.message_field(2) - - -@dataclass -class TemporaryRelation(betterproto.Message): - """E029""" - - key: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class TemporaryRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "TemporaryRelation" = betterproto.message_field(2) - - -@dataclass -class RenameSchema(betterproto.Message): - """E030""" - - old_key: "ReferenceKeyMsg" = betterproto.message_field(1) - new_key: "ReferenceKeyMsg" = betterproto.message_field(2) - - -@dataclass -class RenameSchemaMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "RenameSchema" = betterproto.message_field(2) - - -@dataclass -class DumpBeforeAddGraph(betterproto.Message): +class CacheDumpGraph(betterproto.Message): """E031""" dump: Dict[str, "ListOfStrings"] = betterproto.map_field( 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE ) + before_after: str = betterproto.string_field(2) + action: str = betterproto.string_field(3) @dataclass -class DumpBeforeAddGraphMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DumpBeforeAddGraph" = betterproto.message_field(2) - - -@dataclass -class DumpAfterAddGraph(betterproto.Message): - """E032""" - - dump: Dict[str, "ListOfStrings"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE - ) - - -@dataclass -class DumpAfterAddGraphMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DumpAfterAddGraph" = betterproto.message_field(2) - - -@dataclass -class DumpBeforeRenameSchema(betterproto.Message): - """E033""" - - dump: Dict[str, "ListOfStrings"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE - ) - - -@dataclass -class DumpBeforeRenameSchemaMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DumpBeforeRenameSchema" = betterproto.message_field(2) - - -@dataclass -class DumpAfterRenameSchema(betterproto.Message): - """E034""" - - dump: Dict[str, "ListOfStrings"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE - ) - - -@dataclass -class DumpAfterRenameSchemaMsg(betterproto.Message): +class CacheDumpGraphMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DumpAfterRenameSchema" = betterproto.message_field(2) + data: "CacheDumpGraph" = betterproto.message_field(2) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 2e6c0e742ab..b1fb163bc0f 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -593,140 +593,35 @@ message SchemaDropMsg { } // E022 -message UncachedRelation { - ReferenceKeyMsg dep_key = 1; +message CacheAction { + string action = 1; ReferenceKeyMsg ref_key = 2; + ReferenceKeyMsg ref_key_2 = 3; + ReferenceKeyMsg ref_key_3 = 4; + repeated ReferenceKeyMsg ref_list = 5; } -message UncachedRelationMsg { +message CacheActionMsg { EventInfo info = 1; - UncachedRelation data = 2; + CacheAction data = 2; } -// E023 -message AddLink { - ReferenceKeyMsg dep_key = 1; - ReferenceKeyMsg ref_key = 2; -} - -message AddLinkMsg { - EventInfo info = 1; - AddLink data = 2; -} - -// E024 -message AddRelation { - ReferenceKeyMsg relation = 1; -} - -message AddRelationMsg { - EventInfo info = 1; - AddRelation data = 2; -} - -// E025 -message DropMissingRelation { - ReferenceKeyMsg relation = 1; -} - -message DropMissingRelationMsg { - EventInfo info = 1; - DropMissingRelation data = 2; -} - -// E026 -message DropCascade { - ReferenceKeyMsg dropped = 1; - repeated ReferenceKeyMsg consequences = 2; -} - -message DropCascadeMsg { - EventInfo info = 1; - DropCascade data = 2; -} - -// E027 -message DropRelation { - ReferenceKeyMsg dropped = 1; -} - -message DropRelationMsg { - EventInfo info = 1; - DropRelation data = 2; -} - -// E028 -message UpdateReference { - ReferenceKeyMsg old_key = 1; - ReferenceKeyMsg new_key = 2; - ReferenceKeyMsg cached_key = 3; -} - -message UpdateReferenceMsg { - EventInfo info = 1; - UpdateReference data = 2; -} - -// E029 -message TemporaryRelation { - ReferenceKeyMsg key = 1; -} - -message TemporaryRelationMsg { - EventInfo info = 1; - TemporaryRelation data = 2; -} - -// E030 -message RenameSchema { - ReferenceKeyMsg old_key = 1; - ReferenceKeyMsg new_key = 2; -} - -message RenameSchemaMsg { - EventInfo info = 1; - RenameSchema data = 2; -} +// Skipping E023, E024, E025, E026, E027, E028, E029, E0230 // E031 -message DumpBeforeAddGraph { +message CacheDumpGraph { map dump = 1; + string before_after = 2; + string action = 3; } -message DumpBeforeAddGraphMsg { +message CacheDumpGraphMsg { EventInfo info = 1; - DumpBeforeAddGraph data = 2; + CacheDumpGraph data = 2; } -// E032 -message DumpAfterAddGraph { - map dump = 1; -} -message DumpAfterAddGraphMsg { - EventInfo info = 1; - DumpAfterAddGraph data = 2; -} - -// E033 -message DumpBeforeRenameSchema { - map dump = 1; -} - -message DumpBeforeRenameSchemaMsg { - EventInfo info = 1; - DumpBeforeRenameSchema data = 2; -} - -// E034 -message DumpAfterRenameSchema { - map dump = 1; -} - -message DumpAfterRenameSchemaMsg { - EventInfo info = 1; - DumpAfterRenameSchema data = 2; -} +// Skipping E032, E033, E034 // E035 message AdapterImportError { diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 1066ab6b5d3..a5d344d9977 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -592,130 +592,54 @@ def message(self) -> str: return f'Dropping schema "{self.relation}".' -# TODO pretty sure this is only ever called in dead code -# see: core/dbt/adapters/cache.py _add_link vs add_link @dataclass -class UncachedRelation(DebugLevel, Cache, pt.UncachedRelation): +class CacheAction(DebugLevel, Cache, pt.CacheAction): def code(self): return "E022" - def message(self) -> str: - return ( - f"{self.dep_key} references {str(self.ref_key)} " - f"but {self.ref_key.database}.{self.ref_key.schema}" - "is not in the cache, skipping assumed external relation" - ) - - -@dataclass -class AddLink(DebugLevel, Cache, pt.AddLink): - def code(self): - return "E023" - - def message(self) -> str: - return f"adding link, {self.dep_key} references {self.ref_key}" - - -@dataclass -class AddRelation(DebugLevel, Cache, pt.AddRelation): - def code(self): - return "E024" - - def message(self) -> str: - return f"Adding relation: {str(self.relation)}" - - -@dataclass -class DropMissingRelation(DebugLevel, Cache, pt.DropMissingRelation): - def code(self): - return "E025" - - def message(self) -> str: - return f"dropped a nonexistent relationship: {str(self.relation)}" - - -@dataclass -class DropCascade(DebugLevel, Cache, pt.DropCascade): - def code(self): - return "E026" - - def message(self) -> str: - return f"drop {self.dropped} is cascading to {self.consequences}" - - -@dataclass -class DropRelation(DebugLevel, Cache, pt.DropRelation): - def code(self): - return "E027" - - def message(self) -> str: - return f"Dropping relation: {self.dropped}" - - -@dataclass -class UpdateReference(DebugLevel, Cache, pt.UpdateReference): - def code(self): - return "E028" - - def message(self) -> str: - return ( - f"updated reference from {self.old_key} -> {self.cached_key} to " - f"{self.new_key} -> {self.cached_key}" - ) - - -@dataclass -class TemporaryRelation(DebugLevel, Cache, pt.TemporaryRelation): - def code(self): - return "E029" - - def message(self) -> str: - return f"old key {self.key} not found in self.relations, assuming temporary" - + def message(self): + if self.action == "add_link": + return f"adding link, {self.ref_key} references {self.ref_key_2}" + elif self.action == "add_relation": + return f"adding relation: {str(self.ref_key)}" + elif self.action == "drop_missing_relation": + return f"dropped a nonexistent relationship: {str(self.ref_key)}" + elif self.action == "drop_cascade": + return f"drop {self.ref_key} is cascading to {self.ref_list}" + elif self.action == "drop_relation": + return f"Dropping relation: {self.ref_key}" + elif self.action == "update_reference": + return ( + f"updated reference from {self.ref_key} -> {self.ref_key_3} to " + f"{self.ref_key_2} -> {self.ref_key_3}" + ) + elif self.action == "temporary_relation": + return f"old key {self.ref_key} not found in self.relations, assuming temporary" + elif self.action == "rename_relation": + return f"Renaming relation {self.ref_key} to {self.ref_key_2}" + elif self.action == "uncached_relation": + return ( + f"{self.ref_key_2} references {str(self.ref_key)} " + f"but {self.ref_key.database}.{self.ref_key.schema}" + "is not in the cache, skipping assumed external relation" + ) + else: + return f"{self.ref_key}" -@dataclass -class RenameSchema(DebugLevel, Cache, pt.RenameSchema): - def code(self): - return "E030" - def message(self) -> str: - return f"Renaming relation {self.old_key} to {self.new_key}" +# Skipping E023, E024, E025, E026, E027, E028, E029, E030 @dataclass -class DumpBeforeAddGraph(DebugLevel, Cache, pt.DumpBeforeAddGraph): +class CacheDumpGraph(DebugLevel, Cache, pt.CacheDumpGraph): def code(self): return "E031" def message(self) -> str: - return f"before adding : {self.dump}" + return f"{self.before_after} {self.action} : {self.dump}" -@dataclass -class DumpAfterAddGraph(DebugLevel, Cache, pt.DumpAfterAddGraph): - def code(self): - return "E032" - - def message(self) -> str: - return f"after adding: {self.dump}" - - -@dataclass -class DumpBeforeRenameSchema(DebugLevel, Cache, pt.DumpBeforeRenameSchema): - def code(self): - return "E033" - - def message(self) -> str: - return f"before rename: {self.dump}" - - -@dataclass -class DumpAfterRenameSchema(DebugLevel, Cache, pt.DumpAfterRenameSchema): - def code(self): - return "E034" - - def message(self) -> str: - return f"after rename: {self.dump}" +# Skipping E032, E033, E034 @dataclass diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 112155ae42c..a50098e23c1 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -158,35 +158,12 @@ def test_event_codes(self): ), SchemaCreation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), SchemaDrop(relation=ReferenceKeyMsg(database="", schema="", identifier="")), - UncachedRelation( - dep_key=ReferenceKeyMsg(database="", schema="", identifier=""), + CacheAction( + action="adding_relation", ref_key=ReferenceKeyMsg(database="", schema="", identifier=""), + ref_key_2=ReferenceKeyMsg(database="", schema="", identifier=""), ), - AddLink( - dep_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ref_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ), - AddRelation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), - DropMissingRelation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), - DropCascade( - dropped=ReferenceKeyMsg(database="", schema="", identifier=""), - consequences=[ReferenceKeyMsg(database="", schema="", identifier="")], - ), - DropRelation(dropped=ReferenceKeyMsg()), - UpdateReference( - old_key=ReferenceKeyMsg(database="", schema="", identifier=""), - new_key=ReferenceKeyMsg(database="", schema="", identifier=""), - cached_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ), - TemporaryRelation(key=ReferenceKeyMsg(database="", schema="", identifier="")), - RenameSchema( - old_key=ReferenceKeyMsg(database="", schema="", identifier=""), - new_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ), - DumpBeforeAddGraph(dump=dict()), - DumpAfterAddGraph(dump=dict()), - DumpBeforeRenameSchema(dump=dict()), - DumpAfterRenameSchema(dump=dict()), + CacheDumpGraph(before_after="before", action="rename", dump=dict()), AdapterImportError(exc=""), PluginLoadError(exc_info=""), NewConnectionOpening(connection_state=""), From c06dbb92ba9f4d37504efc5ce333db6f8764d751 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Jan 2023 11:42:45 -0500 Subject: [PATCH 4/7] Changie --- .changes/unreleased/Under the Hood-20230110-114233.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20230110-114233.yaml diff --git a/.changes/unreleased/Under the Hood-20230110-114233.yaml b/.changes/unreleased/Under the Hood-20230110-114233.yaml new file mode 100644 index 00000000000..0b998f9e31b --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230110-114233.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Combine some logging events +time: 2023-01-10T11:42:33.580756-05:00 +custom: + Author: gshank + Issue: 1716 1718 1719 From 1eff853e08328369b1227d1fc1e301d6505ee1e2 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Jan 2023 11:47:41 -0500 Subject: [PATCH 5/7] fix ticket number --- .changes/unreleased/Under the Hood-20230110-114233.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Under the Hood-20230110-114233.yaml b/.changes/unreleased/Under the Hood-20230110-114233.yaml index 0b998f9e31b..c18a26d4a03 100644 --- a/.changes/unreleased/Under the Hood-20230110-114233.yaml +++ b/.changes/unreleased/Under the Hood-20230110-114233.yaml @@ -3,4 +3,4 @@ body: Combine some logging events time: 2023-01-10T11:42:33.580756-05:00 custom: Author: gshank - Issue: 1716 1718 1719 + Issue: 1716 1717 1719 From 73547f92602a91ef5a18bf784344c931587768a7 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Jan 2023 12:13:01 -0500 Subject: [PATCH 6/7] Ooops. Add another file. --- core/dbt/events/types.proto | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 5df72ffbea2..b1fb163bc0f 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -347,11 +347,6 @@ message FunctionDeprecatedMsg { FunctionDeprecated data = 2; } -message FunctionDeprecatedMsg { - EventInfo info = 1; - FunctionDeprecated data = 2; -} - // E - DB Adapter // E001 From eb47eb1c7e1c337eca98c60b14f9f74c0958b02b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Jan 2023 13:36:26 -0500 Subject: [PATCH 7/7] fix serialization of profile names --- core/dbt/events/proto_types.py | 12 ++++++------ core/dbt/events/types.proto | 12 ++++++------ core/dbt/events/types.py | 4 ++-- core/dbt/task/base.py | 10 +++++----- tests/unit/test_events.py | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 31089ad3fb4..32be83ef324 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -171,20 +171,20 @@ class InvalidVarsYAMLMsg(betterproto.Message): @dataclass -class DbtProjectError(betterproto.Message): +class LogDbtProjectError(betterproto.Message): """A009""" exc: str = betterproto.string_field(1) @dataclass -class DbtProjectErrorMsg(betterproto.Message): +class LogDbtProjectErrorMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DbtProjectError" = betterproto.message_field(2) + data: "LogDbtProjectError" = betterproto.message_field(2) @dataclass -class DbtProfileError(betterproto.Message): +class LogDbtProfileError(betterproto.Message): """A011""" exc: str = betterproto.string_field(1) @@ -192,9 +192,9 @@ class DbtProfileError(betterproto.Message): @dataclass -class DbtProfileErrorMsg(betterproto.Message): +class LogDbtProfileErrorMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DbtProfileError" = betterproto.message_field(2) + data: "LogDbtProfileError" = betterproto.message_field(2) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index b1fb163bc0f..178780f3f85 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -132,26 +132,26 @@ message InvalidVarsYAMLMsg { } // A009 -message DbtProjectError { +message LogDbtProjectError { string exc = 1; } -message DbtProjectErrorMsg { +message LogDbtProjectErrorMsg { EventInfo info = 1; - DbtProjectError data = 2; + LogDbtProjectError data = 2; } // Skipped A010 // A011 -message DbtProfileError { +message LogDbtProfileError { string exc = 1; repeated string profiles = 2; } -message DbtProfileErrorMsg { +message LogDbtProfileErrorMsg { EventInfo info = 1; - DbtProfileError data = 2; + LogDbtProfileError data = 2; } // Skipped A012, A013, A014, A015, A016 diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index edd83d052b0..b76188a8c97 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -117,7 +117,7 @@ def message(self) -> str: @dataclass -class DbtProjectError(ErrorLevel, pt.DbtProjectError): +class LogDbtProjectError(ErrorLevel, pt.LogDbtProjectError): def code(self): return "A009" @@ -132,7 +132,7 @@ def message(self) -> str: @dataclass -class DbtProfileError(ErrorLevel, pt.DbtProfileError): +class LogDbtProfileError(ErrorLevel, pt.LogDbtProfileError): def code(self): return "A011" diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 0e4d93b688a..b7ababdd067 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -24,8 +24,8 @@ from dbt.logger import log_manager from dbt.events.functions import fire_event from dbt.events.types import ( - DbtProjectError, - DbtProfileError, + LogDbtProjectError, + LogDbtProfileError, CatchableExceptionOnRun, InternalExceptionOnRun, GenericExceptionOnRun, @@ -96,13 +96,13 @@ def from_args(cls, args): # for the clean or deps tasks config = cls.ConfigType.from_args(args) except dbt.exceptions.DbtProjectError as exc: - fire_event(DbtProjectError(exc=str(exc))) + fire_event(LogDbtProjectError(exc=str(exc))) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc except dbt.exceptions.DbtProfileError as exc: - all_profiles = read_profiles(flags.PROFILES_DIR).keys() - fire_event(DbtProfileError(exc=str(exc), profiles=all_profiles)) + all_profile_names = list(read_profiles(flags.PROFILES_DIR).keys()) + fire_event(LogDbtProfileError(exc=str(exc), profiles=all_profile_names)) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc return cls(args, config) diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index a50098e23c1..77435a5a26e 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -111,8 +111,8 @@ def test_event_codes(self): MergedFromState(num_merged=0, sample=[]), MissingProfileTarget(profile_name="", target_name=""), InvalidVarsYAML(), - DbtProjectError(), - DbtProfileError(), + LogDbtProjectError(), + LogDbtProfileError(), StarterProjectPath(dir=""), ConfigFolderDirectory(dir=""), NoSampleProfileFound(adapter=""),