Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logging to materialize #1467

Merged
merged 2 commits into from
Apr 14, 2021
Merged

Add logging to materialize #1467

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
25a1c54
Add logging to materialize
woop Apr 14, 2021
f171f4c
Fix linting
woop Apr 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion sdk/python/feast/feature_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from typing import Any, Dict, List, Optional, Tuple, Union

import pandas as pd
from colorama import Fore, Style

from feast import utils
from feast.entity import Entity
Expand Down Expand Up @@ -327,13 +328,16 @@ def materialize_incremental(
if start_date is None:
if feature_view.ttl is None:
raise Exception(
f"No start time found for feature view {feature_view.name}. materialize_incremental() requires either a ttl to be set or for materialize() to have been run at least once."
f"No start time found for feature view {feature_view.name}. materialize_incremental() requires"
f" either a ttl to be set or for materialize() to have been run at least once."
)
start_date = datetime.utcnow() - feature_view.ttl
provider = self._get_provider()
_print_materialization_log(start_date, end_date, feature_view)
provider.materialize_single_feature_view(
feature_view, start_date, end_date, self._registry, self.project
)
print(" done!")

def materialize(
self,
Expand Down Expand Up @@ -387,9 +391,11 @@ def materialize(
# TODO paging large loads
for feature_view in feature_views_to_materialize:
provider = self._get_provider()
_print_materialization_log(start_date, end_date, feature_view)
provider.materialize_single_feature_view(
feature_view, start_date, end_date, self._registry, self.project
)
print(" done!")

def get_online_features(
self, feature_refs: List[str], entity_rows: List[Dict[str, Any]],
Expand Down Expand Up @@ -574,3 +580,13 @@ def _get_table_entity_keys(
EntityKeyProto(join_keys=entity_names, entity_values=entity_values)
)
return entity_key_protos


def _print_materialization_log(start_date, end_date, feature_view):
print(
f"Materializing feature view {Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}"
f" from {Style.BRIGHT + Fore.GREEN}{start_date.astimezone()}{Style.RESET_ALL}"
f" to {Style.BRIGHT + Fore.GREEN}{end_date.astimezone()}{Style.RESET_ALL}",
end="",
flush=True,
)