Skip to content

Commit

Permalink
Add native types for Python SDK online retrieval (#826)
Browse files Browse the repository at this point in the history 
* Add native types for get_online_features

* Address PR comments

* Address PR comments

* Fix lint

* Fix test order

* Address PR comments

* Address PR comments

* Address PR comments

* Update docs and example notebooks

* Update docs

* Fix lint

* Address PR comments

* Add bugfix

* Update wrong import

* Move warning into condition check

* Params reformatting

* Refactor type inference in online retriever

* Fix typo in variable name

Co-authored-by: Willem Pienaar <[email protected]>
  • Loading branch information
@terryyylim @woop
terryyylim and woop authored Jun 30, 2020
1 parent 0f5bd69 commit 9f35706
Show file tree
Hide file tree
Showing 7 changed files with 762 additions and 262 deletions.
203 changes: 115 additions & 88 deletions examples/basic/basic.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,48 +148,48 @@
"output_type": "stream",
"text": [
" datetime customer_id daily_transactions \\\n",
"0 2020-06-09 00:00:00+00:00 1001 0.564751 \n",
"1 2020-06-09 00:00:00+00:00 1002 3.945566 \n",
"2 2020-06-09 00:00:00+00:00 1003 7.291928 \n",
"3 2020-06-09 00:00:00+00:00 1004 6.690477 \n",
"4 2020-06-09 00:00:00+00:00 1005 6.415899 \n",
"5 2020-06-10 00:00:00+00:00 1001 0.347294 \n",
"6 2020-06-10 00:00:00+00:00 1002 5.363853 \n",
"7 2020-06-10 00:00:00+00:00 1003 0.538129 \n",
"8 2020-06-10 00:00:00+00:00 1004 4.755425 \n",
"9 2020-06-10 00:00:00+00:00 1005 2.867527 \n",
"10 2020-06-11 00:00:00+00:00 1001 9.493098 \n",
"11 2020-06-11 00:00:00+00:00 1002 5.130665 \n",
"12 2020-06-11 00:00:00+00:00 1003 1.794191 \n",
"13 2020-06-11 00:00:00+00:00 1004 4.698504 \n",
"14 2020-06-11 00:00:00+00:00 1005 2.908603 \n",
"15 2020-06-12 00:00:00+00:00 1001 9.857894 \n",
"16 2020-06-12 00:00:00+00:00 1002 5.416553 \n",
"17 2020-06-12 00:00:00+00:00 1003 5.374058 \n",
"18 2020-06-12 00:00:00+00:00 1004 9.834441 \n",
"19 2020-06-12 00:00:00+00:00 1005 0.480373 \n",
"0 2020-06-17 00:00:00+00:00 1001 4.900417 \n",
"1 2020-06-17 00:00:00+00:00 1002 7.440329 \n",
"2 2020-06-17 00:00:00+00:00 1003 4.224760 \n",
"3 2020-06-17 00:00:00+00:00 1004 5.482722 \n",
"4 2020-06-17 00:00:00+00:00 1005 2.200896 \n",
"5 2020-06-18 00:00:00+00:00 1001 8.173628 \n",
"6 2020-06-18 00:00:00+00:00 1002 3.164327 \n",
"7 2020-06-18 00:00:00+00:00 1003 7.248387 \n",
"8 2020-06-18 00:00:00+00:00 1004 9.274397 \n",
"9 2020-06-18 00:00:00+00:00 1005 7.846449 \n",
"10 2020-06-19 00:00:00+00:00 1001 9.028874 \n",
"11 2020-06-19 00:00:00+00:00 1002 5.140390 \n",
"12 2020-06-19 00:00:00+00:00 1003 4.537877 \n",
"13 2020-06-19 00:00:00+00:00 1004 6.797491 \n",
"14 2020-06-19 00:00:00+00:00 1005 8.234574 \n",
"15 2020-06-20 00:00:00+00:00 1001 8.319164 \n",
"16 2020-06-20 00:00:00+00:00 1002 7.158817 \n",
"17 2020-06-20 00:00:00+00:00 1003 4.920308 \n",
"18 2020-06-20 00:00:00+00:00 1004 7.974404 \n",
"19 2020-06-20 00:00:00+00:00 1005 2.298012 \n",
"\n",
" total_transactions \n",
"0 73 \n",
"1 75 \n",
"2 95 \n",
"3 50 \n",
"4 65 \n",
"5 28 \n",
"6 76 \n",
"7 42 \n",
"0 45 \n",
"1 77 \n",
"2 8 \n",
"3 40 \n",
"4 53 \n",
"5 33 \n",
"6 93 \n",
"7 68 \n",
"8 53 \n",
"9 61 \n",
"10 86 \n",
"11 31 \n",
"12 69 \n",
"13 9 \n",
"14 51 \n",
"15 23 \n",
"16 1 \n",
"17 34 \n",
"18 13 \n",
"19 50 \n"
"9 11 \n",
"10 19 \n",
"11 2 \n",
"12 1 \n",
"13 59 \n",
"14 95 \n",
"15 37 \n",
"16 93 \n",
"17 73 \n",
"18 46 \n",
"19 12 \n"
]
}
],
Expand Down Expand Up @@ -282,7 +282,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Feature set updated: \"customer_transactions\"\n"
"Feature set created: \"customer_transactions\"\n"
]
}
],
Expand Down Expand Up @@ -310,27 +310,26 @@
" ],\n",
" \"features\": [\n",
" {\n",
" \"name\": \"total_transactions\",\n",
" \"valueType\": \"INT64\"\n",
" },\n",
" {\n",
" \"name\": \"daily_transactions\",\n",
" \"valueType\": \"DOUBLE\"\n",
" },\n",
" {\n",
" \"name\": \"total_transactions\",\n",
" \"valueType\": \"INT64\"\n",
" }\n",
" ],\n",
" \"maxAge\": \"0s\",\n",
" \"source\": {\n",
" \"type\": \"KAFKA\",\n",
" \"kafkaSourceConfig\": {\n",
" \"bootstrapServers\": \"kafka:9092,localhost:9094\",\n",
" \"bootstrapServers\": \"localhost:9094\",\n",
" \"topic\": \"feast-features\"\n",
" }\n",
" },\n",
" \"project\": \"default\"\n",
" },\n",
" \"meta\": {\n",
" \"createdTimestamp\": \"2020-06-18T12:04:08Z\",\n",
" \"status\": \"STATUS_READY\"\n",
" \"createdTimestamp\": \"2020-06-26T12:27:17Z\",\n",
" \"status\": \"STATUS_PENDING\"\n",
" }\n",
"}\n"
]
Expand All @@ -357,17 +356,9 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
" 0%| | 0/50 [00:00<?, ?rows/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
Expand All @@ -379,7 +370,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 50/50 [00:01<00:00, 48.83rows/s]"
"100%|██████████| 50/50 [00:01<00:00, 47.23rows/s]"
]
},
{
Expand All @@ -403,10 +394,10 @@
{
"data": {
"text/plain": [
"'8da648b7-d6ac-3970-90c6-cf789078c869'"
"'5e650050-f41d-39fc-bc56-d602c4a478d2'"
]
},
"execution_count": 11,
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -431,34 +422,44 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"field_values {\n",
" fields {\n",
" key: \"customer_id\"\n",
" value {\n",
" int64_val: 1001\n",
" }\n",
"[fields {\n",
" key: \"customer_id\"\n",
" value {\n",
" int64_val: 1001\n",
" }\n",
" fields {\n",
" key: \"daily_transactions\"\n",
" value {\n",
" double_val: 5.1228456657485495\n",
" }\n",
"}\n",
"fields {\n",
" key: \"daily_transactions\"\n",
" value {\n",
" double_val: 0.12021977894872915\n",
" }\n",
" fields {\n",
" key: \"total_transactions\"\n",
" value {\n",
" int64_val: 96\n",
" }\n",
"}\n",
"fields {\n",
" key: \"total_transactions\"\n",
" value {\n",
" int64_val: 0\n",
" }\n",
"}\n",
"\n"
"statuses {\n",
" key: \"customer_id\"\n",
" value: PRESENT\n",
"}\n",
"statuses {\n",
" key: \"daily_transactions\"\n",
" value: PRESENT\n",
"}\n",
"statuses {\n",
" key: \"total_transactions\"\n",
" value: PRESENT\n",
"}\n",
"]\n"
]
}
],
Expand All @@ -469,15 +470,41 @@
" f\"total_transactions\",\n",
" ],\n",
" entity_rows=[\n",
" GetOnlineFeaturesRequest.EntityRow(\n",
" fields={\n",
" \"customer_id\": Value(\n",
" int64_val=1001)\n",
" }\n",
" )\n",
" {\n",
" \"customer_id\": 1001\n",
" }\n",
" ],\n",
")\n",
"print(online_features)"
"print(online_features.field_values)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To retrieve the fields in dictionary format, we can utilize `to_dict()` method."
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'daily_transactions': [0.12021977894872915],\n",
" 'total_transactions': [0],\n",
" 'customer_id': [1001]}"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"online_features.to_dict()"
]
},
{
Expand Down Expand Up @@ -708,9 +735,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "local-feast",
"language": "python",
"name": "python3"
"name": "local-feast"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -726,5 +753,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
Loading

0 comments on commit 9f35706

Please sign in to comment.