From e362ba697c6b4868637d1958ab5910f97926c664 Mon Sep 17 00:00:00 2001 From: Bhavin Panchal Date: Wed, 26 Jun 2019 15:38:43 +0100 Subject: [PATCH 1/7] Added subscriber degree marshmallow spec --- .../server/query_schemas/subscriber_degree.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py diff --git a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py new file mode 100644 index 0000000000..fa65e4ed97 --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from marshmallow import Schema, fields, post_load +from marshmallow.validate import OneOf, Length + +from flowmachine.features import SubscriberDegree +from .base_exposed_query import BaseExposedQuery +from .custom_fields import SubscriberSubset + +__all__ = ["SubscriberDegreeSchema", "SubscriberDegreeExposed"] + + +class SubscriberDegreeSchema(Schema): + query_kind = fields.String(validate=OneOf(["subscriber_degree"])) + start_date = fields.Date(required=True) + end_date = fields.Date(required=True) + direction = fields.String( + required=False, validate=OneOf(["in", "out", "both"]), default="both" + ) # TODO: use a globally defined enum for this + subscriber_subset = SubscriberSubset() + + @post_load + def make_query_object(self, params): + return SubscriberDegreeExposed(**params) + + +class SubscriberDegreeExposed(BaseExposedQuery): + def __init__(self, *, start_date, end_date, direction, subscriber_subset=None): + # Note: all input parameters need to be defined as attributes on `self` + # so that marshmallow can serialise the object correctly. + self.start_date = start_date + self.end_date = end_date + self.direction = direction + self.subscriber_subset = subscriber_subset + + @property + def _flowmachine_query_obj(self): + """ + Return the underlying flowmachine subscriber_degree object. + + Returns + ------- + Query + """ + return SubscriberDegree( + start=self.start_date, + stop=self.end_date, + direction=self.direction, + subscriber_subset=self.subscriber_subset, + ) From 491ce845e7bd5e68b2c86ce4bbde80214ccfd776 Mon Sep 17 00:00:00 2001 From: bhavin panchal Date: Thu, 27 Jun 2019 12:13:28 +0100 Subject: [PATCH 2/7] Changes for issue 969 --- CHANGELOG.md | 1 + flowclient/flowclient/__init__.py | 2 ++ flowclient/flowclient/client.py | 36 +++++++++++++++++++ .../query_schemas/joined_spatial_aggregate.py | 2 ++ .../tests/query_tests/test_queries.py | 17 +++++++++ 5 files changed, 58 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56c348a6fb..ee9e3fb96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - The dev provisioning Ansible playbook now automatically generates an SSH key pair for the `flowkit` user. [#892](https://github.com/Flowminder/FlowKit/issues/892) - FlowAPI's 'joined_spatial_aggregate' endpoint now exposes unique location counts.[#949](https://github.com/Flowminder/FlowKit/issues/949) +- FlowAPI's 'joined_spatial_aggregate' endpoint now exposes subscriber degree.[#969](https://github.com/Flowminder/FlowKit/issues/969) ### Changed diff --git a/flowclient/flowclient/__init__.py b/flowclient/flowclient/__init__.py index 070f4bc98b..b906a861ae 100644 --- a/flowclient/flowclient/__init__.py +++ b/flowclient/flowclient/__init__.py @@ -36,6 +36,7 @@ joined_spatial_aggregate, radius_of_gyration, unique_location_counts, + subscriber_degree, ) __all__ = [ @@ -64,4 +65,5 @@ "joined_spatial_aggregate", "radius_of_gyration", "unique_location_counts", + "subscriber_degree", ] diff --git a/flowclient/flowclient/client.py b/flowclient/flowclient/client.py index cb29e3293e..65e5af6153 100644 --- a/flowclient/flowclient/client.py +++ b/flowclient/flowclient/client.py @@ -1285,3 +1285,39 @@ def unique_location_counts( "aggregation_unit": aggregation_unit, "subscriber_subset": subscriber_subset, } + + +def subscriber_degree( + *, + start: str, + stop: str, + direction: str, + subscriber_subset: Union[dict, None] = None, +) -> dict: + """ + Return query spec for subscriber degree + + Parameters + ---------- + start : str + ISO format date of the first day of the count, e.g. "2016-01-01" + stop : str + ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" + direction : {"in", "out", "both"}, default "both" + Optionally, include only ingoing or outbound calls/texts. Can be one of "in", "out" or "both". + subscriber_subset : dict or None, default None + Subset of subscribers to include in event counts. Must be None + (= all subscribers) or a dictionary with the specification of a + subset query. + Returns + ------- + dict + Dict which functions as the query specification + """ + return { + "query_kind": "subscriber_degree", + "start": start, + "stop": stop, + "direction": direction, + "subscriber_subset": subscriber_subset, + } diff --git a/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py b/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py index 4e82c31605..484e208c58 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py +++ b/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py @@ -9,6 +9,7 @@ from flowmachine.core.server.query_schemas.radius_of_gyration import ( RadiusOfGyrationSchema, ) +from flowmachine.core.server.query_schemas.subscriber_degree import SubscriberDegree from flowmachine.core.server.query_schemas.unique_location_counts import ( UniqueLocationCountsSchema, ) @@ -27,6 +28,7 @@ class JoinableMetrics(OneOfSchema): type_schemas = { "radius_of_gyration": RadiusOfGyrationSchema, "unique_location_counts": UniqueLocationCountsSchema, + "subscriber_degree": SubscriberDegree, } diff --git a/integration_tests/tests/query_tests/test_queries.py b/integration_tests/tests/query_tests/test_queries.py index 6e79ae9413..f20c7b870c 100644 --- a/integration_tests/tests/query_tests/test_queries.py +++ b/integration_tests/tests/query_tests/test_queries.py @@ -108,6 +108,23 @@ }, }, ), + ( + "joined_spatial_aggregate", + { + "locations": { + "query_kind": "daily_location", + "date": "2016-01-01", + "aggregation_unit": "admin3", + "method": "last", + }, + "metric": { + "query_kind": "subscriber_degree", + "start": "2016-01-01", + "stop": "2016-01-02", + "direction": "both", + }, + }, + ), ( "joined_spatial_aggregate", { From 56a4167995f9e48ccd1552a656587af843e5e252 Mon Sep 17 00:00:00 2001 From: bhavin panchal Date: Thu, 27 Jun 2019 17:07:31 +0100 Subject: [PATCH 3/7] Changes to approval test and joined spatial aggregate --- .../query_schemas/joined_spatial_aggregate.py | 6 ++- .../server/query_schemas/subscriber_degree.py | 14 +++--- ...t_generated_openapi_json_spec.approved.txt | 43 +++++++++++++++++++ ..._generated_openapi_redoc_spec.approved.txt | 42 ++++++++++++++++++ ...t_generated_openapi_yaml_spec.approved.txt | 43 +++++++++++++++++++ ..._of_flowmachine_query_schemas.approved.txt | 42 ++++++++++++++++++ 6 files changed, 181 insertions(+), 9 deletions(-) diff --git a/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py b/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py index 484e208c58..d180c9949b 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py +++ b/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py @@ -9,7 +9,9 @@ from flowmachine.core.server.query_schemas.radius_of_gyration import ( RadiusOfGyrationSchema, ) -from flowmachine.core.server.query_schemas.subscriber_degree import SubscriberDegree +from flowmachine.core.server.query_schemas.subscriber_degree import ( + SubscriberDegreeSchema, +) from flowmachine.core.server.query_schemas.unique_location_counts import ( UniqueLocationCountsSchema, ) @@ -28,7 +30,7 @@ class JoinableMetrics(OneOfSchema): type_schemas = { "radius_of_gyration": RadiusOfGyrationSchema, "unique_location_counts": UniqueLocationCountsSchema, - "subscriber_degree": SubscriberDegree, + "subscriber_degree": SubscriberDegreeSchema, } diff --git a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py index fa65e4ed97..05464eb2cf 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py +++ b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py @@ -14,8 +14,8 @@ class SubscriberDegreeSchema(Schema): query_kind = fields.String(validate=OneOf(["subscriber_degree"])) - start_date = fields.Date(required=True) - end_date = fields.Date(required=True) + start = fields.Date(required=True) + stop = fields.Date(required=True) direction = fields.String( required=False, validate=OneOf(["in", "out", "both"]), default="both" ) # TODO: use a globally defined enum for this @@ -27,11 +27,11 @@ def make_query_object(self, params): class SubscriberDegreeExposed(BaseExposedQuery): - def __init__(self, *, start_date, end_date, direction, subscriber_subset=None): + def __init__(self, *, start, stop, direction, subscriber_subset=None): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. - self.start_date = start_date - self.end_date = end_date + self.start = start + self.stop = stop self.direction = direction self.subscriber_subset = subscriber_subset @@ -45,8 +45,8 @@ def _flowmachine_query_obj(self): Query """ return SubscriberDegree( - start=self.start_date, - stop=self.end_date, + start=self.start, + stop=self.stop, direction=self.direction, subscriber_subset=self.subscriber_subset, ) diff --git a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt index b86444dda2..525b72ac3f 100644 --- a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt +++ b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt @@ -335,6 +335,7 @@ "discriminator": { "mapping": { "radius_of_gyration": "#/components/schemas/RadiusOfGyration", + "subscriber_degree": "#/components/schemas/SubscriberDegree", "unique_location_counts": "#/components/schemas/UniqueLocationCounts" }, "propertyName": "query_kind" @@ -343,6 +344,9 @@ { "$ref": "#/components/schemas/RadiusOfGyration" }, + { + "$ref": "#/components/schemas/SubscriberDegree" + }, { "$ref": "#/components/schemas/UniqueLocationCounts" } @@ -842,6 +846,45 @@ ], "type": "object" }, + "SubscriberDegree": { + "properties": { + "direction": { + "enum": [ + "both", + "in", + "out" + ], + "type": "string" + }, + "query_kind": { + "enum": [ + "subscriber_degree" + ], + "type": "string" + }, + "start": { + "format": "date", + "type": "string" + }, + "stop": { + "format": "date", + "type": "string" + }, + "subscriber_subset": { + "enum": [ + null + ], + "nullable": true, + "type": "string" + } + }, + "required": [ + "query_kind", + "start", + "stop" + ], + "type": "object" + }, "TotalNetworkObjects": { "properties": { "aggregation_unit": { diff --git a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_redoc_spec.approved.txt b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_redoc_spec.approved.txt index 6b1c92f6aa..dd41e417d3 100644 --- a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_redoc_spec.approved.txt +++ b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_redoc_spec.approved.txt @@ -291,6 +291,9 @@ { "$ref": "#/components/schemas/RadiusOfGyration" }, + { + "$ref": "#/components/schemas/SubscriberDegree" + }, { "$ref": "#/components/schemas/UniqueLocationCounts" } @@ -790,6 +793,45 @@ ], "type": "object" }, + "SubscriberDegree": { + "properties": { + "direction": { + "enum": [ + "both", + "in", + "out" + ], + "type": "string" + }, + "query_kind": { + "enum": [ + "subscriber_degree" + ], + "type": "string" + }, + "start": { + "format": "date", + "type": "string" + }, + "stop": { + "format": "date", + "type": "string" + }, + "subscriber_subset": { + "enum": [ + null + ], + "nullable": true, + "type": "string" + } + }, + "required": [ + "query_kind", + "start", + "stop" + ], + "type": "object" + }, "TotalNetworkObjects": { "properties": { "aggregation_unit": { diff --git a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_yaml_spec.approved.txt b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_yaml_spec.approved.txt index b86444dda2..525b72ac3f 100644 --- a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_yaml_spec.approved.txt +++ b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_yaml_spec.approved.txt @@ -335,6 +335,7 @@ "discriminator": { "mapping": { "radius_of_gyration": "#/components/schemas/RadiusOfGyration", + "subscriber_degree": "#/components/schemas/SubscriberDegree", "unique_location_counts": "#/components/schemas/UniqueLocationCounts" }, "propertyName": "query_kind" @@ -343,6 +344,9 @@ { "$ref": "#/components/schemas/RadiusOfGyration" }, + { + "$ref": "#/components/schemas/SubscriberDegree" + }, { "$ref": "#/components/schemas/UniqueLocationCounts" } @@ -842,6 +846,45 @@ ], "type": "object" }, + "SubscriberDegree": { + "properties": { + "direction": { + "enum": [ + "both", + "in", + "out" + ], + "type": "string" + }, + "query_kind": { + "enum": [ + "subscriber_degree" + ], + "type": "string" + }, + "start": { + "format": "date", + "type": "string" + }, + "stop": { + "format": "date", + "type": "string" + }, + "subscriber_subset": { + "enum": [ + null + ], + "nullable": true, + "type": "string" + } + }, + "required": [ + "query_kind", + "start", + "stop" + ], + "type": "object" + }, "TotalNetworkObjects": { "properties": { "aggregation_unit": { diff --git a/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt b/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt index 80489b8534..6be20c51fb 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt +++ b/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt @@ -327,6 +327,7 @@ "discriminator": { "mapping": { "radius_of_gyration": "#/components/schemas/RadiusOfGyration", + "subscriber_degree": "#/components/schemas/SubscriberDegree", "unique_location_counts": "#/components/schemas/UniqueLocationCounts" }, "propertyName": "query_kind" @@ -335,6 +336,9 @@ { "$ref": "#/components/schemas/RadiusOfGyration" }, + { + "$ref": "#/components/schemas/SubscriberDegree" + }, { "$ref": "#/components/schemas/UniqueLocationCounts" } @@ -825,6 +829,44 @@ ], "type": "object" }, + "SubscriberDegree": { + "properties": { + "direction": { + "enum": [ + "both", + "in", + "out" + ], + "type": "string" + }, + "query_kind": { + "enum": [ + "subscriber_degree" + ], + "type": "string" + }, + "start": { + "format": "date", + "type": "string" + }, + "stop": { + "format": "date", + "type": "string" + }, + "subscriber_subset": { + "enum": [ + null + ], + "nullable": true, + "type": "string" + } + }, + "required": [ + "start", + "stop" + ], + "type": "object" + }, "TotalNetworkObjects": { "properties": { "aggregation_unit": { From 5d9d7fb52022c945f0e579223a36110121f23ea5 Mon Sep 17 00:00:00 2001 From: bhavin panchal Date: Thu, 27 Jun 2019 18:09:54 +0100 Subject: [PATCH 4/7] Changes in test --- .../tests/query_tests/test_queries.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/integration_tests/tests/query_tests/test_queries.py b/integration_tests/tests/query_tests/test_queries.py index f20c7b870c..542b1e8e4c 100644 --- a/integration_tests/tests/query_tests/test_queries.py +++ b/integration_tests/tests/query_tests/test_queries.py @@ -111,18 +111,12 @@ ( "joined_spatial_aggregate", { - "locations": { - "query_kind": "daily_location", - "date": "2016-01-01", - "aggregation_unit": "admin3", - "method": "last", - }, - "metric": { - "query_kind": "subscriber_degree", - "start": "2016-01-01", - "stop": "2016-01-02", - "direction": "both", - }, + "locations": flowclient.daily_location( + date="2016-01-01", aggregation_unit="admin3", method="last" + ), + "metric": flowclient.radius_of_gyration( + start_date="2016-01-01", end_date="2016-01-02" + ), }, ), ( @@ -131,8 +125,8 @@ "locations": flowclient.daily_location( date="2016-01-01", aggregation_unit="admin3", method="last" ), - "metric": flowclient.radius_of_gyration( - start_date="2016-01-01", end_date="2016-01-02" + "metric": flowclient.subscriber_degree( + start="2016-01-01", stop="2016-01-02", direction="both" ), }, ), From 5df5d72ce9517d3beac24ee16603720f5db6eb29 Mon Sep 17 00:00:00 2001 From: bhavin panchal Date: Fri, 28 Jun 2019 10:01:16 +0100 Subject: [PATCH 5/7] Changes in marshmallow schema --- .../flowmachine/core/server/query_schemas/subscriber_degree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py index 05464eb2cf..82cd14942b 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py +++ b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py @@ -22,7 +22,7 @@ class SubscriberDegreeSchema(Schema): subscriber_subset = SubscriberSubset() @post_load - def make_query_object(self, params): + def make_query_object(self, **params): return SubscriberDegreeExposed(**params) From b6553c59ffa3c9391e23afce2137e1c21ade38c7 Mon Sep 17 00:00:00 2001 From: bhavin panchal Date: Fri, 28 Jun 2019 10:20:52 +0100 Subject: [PATCH 6/7] Added kwargs in marshmallow --- .../flowmachine/core/server/query_schemas/subscriber_degree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py index 82cd14942b..ec2f259b5b 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py +++ b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py @@ -22,7 +22,7 @@ class SubscriberDegreeSchema(Schema): subscriber_subset = SubscriberSubset() @post_load - def make_query_object(self, **params): + def make_query_object(self, params, **kwargs): return SubscriberDegreeExposed(**params) From 19e32df56484316719c877e73cd16bfdadbdd1ba Mon Sep 17 00:00:00 2001 From: bhavin panchal Date: Mon, 1 Jul 2019 10:10:28 +0100 Subject: [PATCH 7/7] Requested change for subscriber function --- flowclient/flowclient/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowclient/flowclient/client.py b/flowclient/flowclient/client.py index 65e5af6153..3192ba5cdd 100644 --- a/flowclient/flowclient/client.py +++ b/flowclient/flowclient/client.py @@ -1291,7 +1291,7 @@ def subscriber_degree( *, start: str, stop: str, - direction: str, + direction: str = "both", subscriber_subset: Union[dict, None] = None, ) -> dict: """