From 38692c1b4192b6797e90a67a0224ff982365a62a Mon Sep 17 00:00:00 2001 From: James Harrison Date: Mon, 1 Jun 2020 18:12:45 +0100 Subject: [PATCH 01/14] Add event_types params to all applicable schemas --- .../consecutive_trips_od_matrix.py | 13 +++++++++++-- .../server/query_schemas/daily_location.py | 14 ++++++++++++-- .../core/server/query_schemas/displacement.py | 6 +++++- .../core/server/query_schemas/handset.py | 6 +++++- .../query_schemas/location_introversion.py | 8 +++++++- .../query_schemas/meaningful_locations.py | 18 +++++++++++++++++- .../server/query_schemas/nocturnal_events.py | 6 +++++- .../query_schemas/pareto_interactions.py | 14 ++++++++++++-- .../server/query_schemas/radius_of_gyration.py | 15 +++++++++++++-- .../server/query_schemas/subscriber_degree.py | 14 ++++++++++++-- .../query_schemas/total_network_objects.py | 9 +++++++-- .../server/query_schemas/trips_od_matrix.py | 13 +++++++++++-- .../query_schemas/unique_location_counts.py | 6 +++++- .../server/query_schemas/unique_locations.py | 6 +++++- .../query_schemas/unique_subscriber_counts.py | 7 +++++-- 15 files changed, 132 insertions(+), 23 deletions(-) diff --git a/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py b/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py index 1e91b34f93..cd3802b2e1 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py +++ b/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py @@ -14,7 +14,7 @@ ) from . import BaseExposedQuery from .base_schema import BaseSchema -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnitMixin __all__ = ["ConsecutiveTripsODMatrixSchema", "ConsecutiveTripsODMatrixExposed"] @@ -22,13 +22,20 @@ class ConsecutiveTripsODMatrixExposed(AggregationUnitMixin, BaseExposedQuery): def __init__( - self, start_date, end_date, *, aggregation_unit, subscriber_subset=None, + self, + start_date, + end_date, + *, + aggregation_unit, + event_types, + subscriber_subset=None, ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types self.subscriber_subset = subscriber_subset @property @@ -46,6 +53,7 @@ def _flowmachine_query_obj(self): self.start_date, self.end_date, spatial_unit=self.aggregation_unit, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) ) @@ -57,6 +65,7 @@ class ConsecutiveTripsODMatrixSchema(AggregationUnitMixin, BaseSchema): query_kind = fields.String(validate=OneOf(["consecutive_trips_od_matrix"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = ConsecutiveTripsODMatrixExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/daily_location.py b/flowmachine/flowmachine/core/server/query_schemas/daily_location.py index 65596cd152..fa766874e7 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/daily_location.py +++ b/flowmachine/flowmachine/core/server/query_schemas/daily_location.py @@ -6,7 +6,7 @@ from marshmallow.validate import OneOf from flowmachine.features import daily_location -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnitMixin from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, @@ -18,13 +18,21 @@ class DailyLocationExposed(BaseExposedQueryWithSampling): def __init__( - self, date, *, method, aggregation_unit, subscriber_subset=None, sampling=None + self, + date, + *, + method, + aggregation_unit, + event_types, + subscriber_subset=None, + sampling=None ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.date = date self.method = method self.aggregation_unit = aggregation_unit + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -41,6 +49,7 @@ def _unsampled_query_obj(self): date=self.date, spatial_unit=self.aggregation_unit, method=self.method, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -50,6 +59,7 @@ class DailyLocationSchema(AggregationUnitMixin, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["daily_location"])) date = ISODateTime(required=True) method = fields.String(required=True, validate=OneOf(["last", "most-common"])) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = DailyLocationExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/displacement.py b/flowmachine/flowmachine/core/server/query_schemas/displacement.py index 1e1215a6ce..509ec02056 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/displacement.py +++ b/flowmachine/flowmachine/core/server/query_schemas/displacement.py @@ -7,7 +7,7 @@ from marshmallow_oneofschema import OneOfSchema from flowmachine.features import Displacement -from .custom_fields import SubscriberSubset, Statistic, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, Statistic, ISODateTime from .daily_location import DailyLocationSchema from .modal_location import ModalLocationSchema from .base_query_with_sampling import ( @@ -35,6 +35,7 @@ def __init__( stop, statistic, reference_location, + event_types, subscriber_subset=None, sampling=None ): @@ -44,6 +45,7 @@ def __init__( self.stop = stop self.statistic = statistic self.reference_location = reference_location + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -61,6 +63,7 @@ def _unsampled_query_obj(self): stop=self.stop, statistic=self.statistic, reference_location=self.reference_location._flowmachine_query_obj, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -71,6 +74,7 @@ class DisplacementSchema(BaseQueryWithSamplingSchema): stop = ISODateTime(required=True) statistic = Statistic() reference_location = fields.Nested(InputToDisplacementSchema, many=False) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = DisplacementExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/handset.py b/flowmachine/flowmachine/core/server/query_schemas/handset.py index 005dcdd745..4dd181f663 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/handset.py +++ b/flowmachine/flowmachine/core/server/query_schemas/handset.py @@ -6,7 +6,7 @@ from marshmallow.validate import OneOf from flowmachine.features import SubscriberHandsetCharacteristic -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, @@ -23,6 +23,7 @@ def __init__( end_date, method, characteristic, + event_types, subscriber_subset=None, sampling=None ): @@ -32,6 +33,7 @@ def __init__( self.end_date = end_date self.method = method self.characteristic = characteristic + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -49,6 +51,7 @@ def _unsampled_query_obj(self): stop=self.end_date, characteristic=self.characteristic, method=self.method, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -64,6 +67,7 @@ class HandsetSchema(BaseQueryWithSamplingSchema): ) ) method = fields.String(validate=OneOf(["last", "most-common"])) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = HandsetExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/location_introversion.py b/flowmachine/flowmachine/core/server/query_schemas/location_introversion.py index d501dc506b..6e0aedfe33 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/location_introversion.py +++ b/flowmachine/flowmachine/core/server/query_schemas/location_introversion.py @@ -9,6 +9,7 @@ from flowmachine.features.location.redacted_location_introversion import ( RedactedLocationIntroversion, ) +from .custom_fields import EventTypes from .base_exposed_query import BaseExposedQuery from .aggregation_unit import AggregationUnitMixin @@ -19,13 +20,16 @@ class LocationIntroversionExposed(BaseExposedQuery): - def __init__(self, *, start_date, end_date, aggregation_unit, direction): + def __init__( + self, *, start_date, end_date, aggregation_unit, direction, event_types + ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit self.direction = direction + self.event_types = event_types @property def _flowmachine_query_obj(self): @@ -42,6 +46,7 @@ def _flowmachine_query_obj(self): stop=self.end_date, spatial_unit=self.aggregation_unit, direction=self.direction, + table=self.event_types, ) ) @@ -54,5 +59,6 @@ class LocationIntroversionSchema(AggregationUnitMixin, BaseSchema): direction = fields.String( required=False, validate=OneOf(["in", "out", "both"]), default="both" ) # TODO: use a globally defined enum for this + event_types = EventTypes() __model__ = LocationIntroversionExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py b/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py index 0790717674..9ef46a2940 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py +++ b/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py @@ -4,7 +4,7 @@ from marshmallow import fields from marshmallow.validate import OneOf -from typing import Union, Dict, List +from typing import Union, Dict, List, Optional from flowmachine.core import make_spatial_unit from flowmachine.features import ( @@ -27,6 +27,7 @@ from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema from .custom_fields import ( + EventTypes, SubscriberSubset, TowerHourOfDayScores, TowerDayOfWeekScores, @@ -53,6 +54,7 @@ def __init__( start_date: str, end_date: str, aggregation_unit: AnySpatialUnit, + event_types: Optional[Union[str, List[str]]], label: str, labels: Dict[str, Dict[str, dict]], tower_day_of_week_scores: Dict[str, float], @@ -66,6 +68,7 @@ def __init__( self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types self.label = label self.labels = labels self.tower_day_of_week_scores = tower_day_of_week_scores @@ -79,6 +82,7 @@ def __init__( labels=labels, start_date=start_date, end_date=end_date, + event_types=event_types, subscriber_subset=subscriber_subset, tower_cluster_call_threshold=tower_cluster_call_threshold, tower_cluster_radius=tower_cluster_radius, @@ -111,6 +115,7 @@ def __init__( start_date: str, end_date: str, aggregation_unit: AnySpatialUnit, + event_types: Optional[Union[str, List[str]]], label_a: str, label_b: str, labels: Dict[str, Dict[str, dict]], @@ -125,6 +130,7 @@ def __init__( self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types self.label_a = label_a self.label_b = label_b self.labels = labels @@ -138,6 +144,7 @@ def __init__( labels=labels, start_date=start_date, end_date=end_date, + event_types=event_types, subscriber_subset=subscriber_subset, tower_cluster_call_threshold=tower_cluster_call_threshold, tower_cluster_radius=tower_cluster_radius, @@ -176,6 +183,7 @@ def __init__( start_date_b: str, end_date_b: str, aggregation_unit: AnySpatialUnit, + event_types: Optional[Union[str, List[str]]], label: str, labels: Dict[str, Dict[str, dict]], tower_day_of_week_scores: Dict[str, float], @@ -191,6 +199,7 @@ def __init__( self.end_date_a = end_date_a self.end_date_b = end_date_b self.aggregation_unit = aggregation_unit + self.event_types = event_types self.label = label self.labels = labels self.tower_day_of_week_scores = tower_day_of_week_scores @@ -202,6 +211,7 @@ def __init__( common_params = dict( labels=labels, label=label, + event_types=event_types, subscriber_subset=subscriber_subset, tower_cluster_call_threshold=tower_cluster_call_threshold, tower_cluster_radius=tower_cluster_radius, @@ -246,6 +256,7 @@ class MeaningfulLocationsAggregateSchema(AggregationUnitMixin, BaseSchema): tower_day_of_week_scores = TowerDayOfWeekScores(required=True) tower_cluster_radius = fields.Float(required=False, default=1.0) tower_cluster_call_threshold = fields.Integer(required=False, default=0) + event_types = EventTypes() subscriber_subset = SubscriberSubset(required=False) __model__ = MeaningfulLocationsAggregateExposed @@ -257,6 +268,7 @@ def _make_meaningful_locations_object( end_date, label, labels, + event_types, subscriber_subset, tower_cluster_call_threshold, tower_cluster_radius, @@ -269,6 +281,7 @@ def _make_meaningful_locations_object( spatial_unit=make_spatial_unit( "versioned-site" ), # note this 'spatial_unit' is not the same as the exposed parameter 'aggregation_unit' + table=event_types, subscriber_subset=subscriber_subset, ) q_call_days = CallDays(subscriber_locations=q_subscriber_locations) @@ -286,6 +299,7 @@ def _make_meaningful_locations_object( spatial_unit=make_spatial_unit( "versioned-site" ), # note this 'spatial_unit' is not the same as the exposed parameter 'aggregation_unit' + table=event_types, subscriber_subset=subscriber_subset, ) q_meaningful_locations = MeaningfulLocations( @@ -309,6 +323,7 @@ class MeaningfulLocationsBetweenLabelODMatrixSchema(AggregationUnitMixin, BaseSc tower_day_of_week_scores = TowerDayOfWeekScores(required=True) tower_cluster_radius = fields.Float(required=False, default=1.0) tower_cluster_call_threshold = fields.Integer(required=False, default=0) + event_types = EventTypes() subscriber_subset = SubscriberSubset(required=False) __model__ = MeaningfulLocationsBetweenLabelODMatrixExposed @@ -330,6 +345,7 @@ class MeaningfulLocationsBetweenDatesODMatrixSchema(AggregationUnitMixin, BaseSc tower_day_of_week_scores = TowerDayOfWeekScores(required=True) tower_cluster_radius = fields.Float(required=False, default=1.0) tower_cluster_call_threshold = fields.Integer(required=False, default=0) + event_types = EventTypes() subscriber_subset = SubscriberSubset(required=False) __model__ = MeaningfulLocationsBetweenDatesODMatrixExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py b/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py index cf93727d57..dd17facb51 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py +++ b/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py @@ -6,7 +6,7 @@ from marshmallow.validate import OneOf, Range from flowmachine.features import NocturnalEvents -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, @@ -23,6 +23,7 @@ def __init__( stop, night_start_hour, night_end_hour, + event_types, subscriber_subset=None, sampling=None ): @@ -31,6 +32,7 @@ def __init__( self.start = start self.stop = stop self.hours = (night_start_hour, night_end_hour) + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -47,6 +49,7 @@ def _unsampled_query_obj(self): start=self.start, stop=self.stop, hours=self.hours, + tables=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -59,6 +62,7 @@ class NocturnalEventsSchema(BaseQueryWithSamplingSchema): validate=Range(0, 23) ) # Tuples aren't supported by apispec https://github.com/marshmallow-code/apispec/issues/399 night_end_hour = fields.Integer(validate=Range(0, 23)) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = NocturnalEventsExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py b/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py index e9f4383cb4..fe2f53c5ea 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py +++ b/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py @@ -6,7 +6,7 @@ from marshmallow.validate import OneOf, Range from flowmachine.features import ParetoInteractions -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, @@ -17,13 +17,21 @@ class ParetoInteractionsExposed(BaseExposedQueryWithSampling): def __init__( - self, *, start, stop, proportion, subscriber_subset=None, sampling=None + self, + *, + start, + stop, + proportion, + event_types, + subscriber_subset=None, + sampling=None ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start = start self.stop = stop self.proportion = proportion + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -40,6 +48,7 @@ def _unsampled_query_obj(self): start=self.start, stop=self.stop, proportion=self.proportion, + tables=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -49,6 +58,7 @@ class ParetoInteractionsSchema(BaseQueryWithSamplingSchema): start = ISODateTime(required=True) stop = ISODateTime(required=True) proportion = fields.Float(required=True, validate=Range(min=0.0, max=1.0)) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = ParetoInteractionsExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py b/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py index 069424ae98..3d1fd1768f 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py +++ b/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py @@ -6,7 +6,7 @@ from marshmallow.validate import OneOf from flowmachine.features import RadiusOfGyration -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, @@ -16,11 +16,20 @@ class RadiusOfGyrationExposed(BaseExposedQueryWithSampling): - def __init__(self, *, start_date, end_date, subscriber_subset=None, sampling=None): + def __init__( + self, + *, + start_date, + end_date, + event_types, + subscriber_subset=None, + sampling=None + ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start_date = start_date self.end_date = end_date + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -36,6 +45,7 @@ def _unsampled_query_obj(self): return RadiusOfGyration( start=self.start_date, stop=self.end_date, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -45,6 +55,7 @@ class RadiusOfGyrationSchema(BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["radius_of_gyration"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = RadiusOfGyrationExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py index 4ab014059a..11134dbd22 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py +++ b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py @@ -6,7 +6,7 @@ from marshmallow.validate import OneOf from flowmachine.features import SubscriberDegree -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, @@ -17,13 +17,21 @@ class SubscriberDegreeExposed(BaseExposedQueryWithSampling): def __init__( - self, *, start, stop, direction, subscriber_subset=None, sampling=None + self, + *, + start, + stop, + direction, + event_types, + subscriber_subset=None, + sampling=None ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start = start self.stop = stop self.direction = direction + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -40,6 +48,7 @@ def _unsampled_query_obj(self): start=self.start, stop=self.stop, direction=self.direction, + tables=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -51,6 +60,7 @@ class SubscriberDegreeSchema(BaseQueryWithSamplingSchema): direction = fields.String( required=False, validate=OneOf(["in", "out", "both"]), default="both" ) # TODO: use a globally defined enum for this + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = SubscriberDegreeExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/total_network_objects.py b/flowmachine/flowmachine/core/server/query_schemas/total_network_objects.py index 8a806818bf..e0942dc550 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/total_network_objects.py +++ b/flowmachine/flowmachine/core/server/query_schemas/total_network_objects.py @@ -8,20 +8,23 @@ from flowmachine.features import TotalNetworkObjects from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema -from .custom_fields import TotalBy, ISODateTime +from .custom_fields import EventTypes, TotalBy, ISODateTime from .aggregation_unit import AggregationUnitMixin __all__ = ["TotalNetworkObjectsSchema", "TotalNetworkObjectsExposed"] class TotalNetworkObjectsExposed(BaseExposedQuery): - def __init__(self, *, start_date, end_date, aggregation_unit, total_by): + def __init__( + self, *, start_date, end_date, aggregation_unit, total_by, event_types + ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit self.total_by = total_by + self.event_types = event_types @property def _flowmachine_query_obj(self): @@ -37,6 +40,7 @@ def _flowmachine_query_obj(self): stop=self.end_date, spatial_unit=self.aggregation_unit, total_by=self.total_by, + table=self.event_types, ) @@ -46,5 +50,6 @@ class TotalNetworkObjectsSchema(AggregationUnitMixin, BaseSchema): start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) total_by = TotalBy(required=False, missing="day") + event_types = EventTypes() __model__ = TotalNetworkObjectsExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py b/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py index e9ade1f7cd..1cf1b1e6fa 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py +++ b/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py @@ -10,7 +10,7 @@ from flowmachine.features.location.trips_od_matrix import TripsODMatrix from . import BaseExposedQuery from .base_schema import BaseSchema -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnitMixin __all__ = ["TripsODMatrixSchema", "TripsODMatrixExposed"] @@ -18,13 +18,20 @@ class TripsODMatrixExposed(BaseExposedQuery): def __init__( - self, start_date, end_date, *, aggregation_unit, subscriber_subset=None, + self, + start_date, + end_date, + *, + aggregation_unit, + event_types, + subscriber_subset=None, ): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types self.subscriber_subset = subscriber_subset @property @@ -42,6 +49,7 @@ def _flowmachine_query_obj(self): self.start_date, self.end_date, spatial_unit=self.aggregation_unit, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) ) @@ -53,6 +61,7 @@ class TripsODMatrixSchema(AggregationUnitMixin, BaseSchema): query_kind = fields.String(validate=OneOf(["trips_od_matrix"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = TripsODMatrixExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py b/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py index 8c7cad6a7c..b85e018657 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py @@ -8,7 +8,7 @@ from flowmachine.features import UniqueLocationCounts from . import BaseExposedQuery from .base_schema import BaseSchema -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnitMixin __all__ = ["UniqueLocationCountsSchema", "UniqueLocationCountsExposed"] @@ -21,6 +21,7 @@ def __init__( start_date, end_date, aggregation_unit, + event_types, subscriber_subset=None, sampling=None ): @@ -29,6 +30,7 @@ def __init__( self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types self.subscriber_subset = subscriber_subset @property @@ -44,6 +46,7 @@ def _flowmachine_query_obj(self): start=self.start_date, stop=self.end_date, spatial_unit=self.aggregation_unit, + tables=self.event_types, subscriber_subset=self.subscriber_subset, ) @@ -52,6 +55,7 @@ class UniqueLocationCountsSchema(AggregationUnitMixin, BaseSchema): query_kind = fields.String(validate=OneOf(["unique_location_counts"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = UniqueLocationCountsExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/unique_locations.py b/flowmachine/flowmachine/core/server/query_schemas/unique_locations.py index 866972992d..c9bf67fd15 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unique_locations.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unique_locations.py @@ -7,7 +7,7 @@ from flowmachine.features import SubscriberLocations from flowmachine.features.subscriber.unique_locations import UniqueLocations -from .custom_fields import SubscriberSubset, ISODateTime +from .custom_fields import EventTypes, SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnitMixin from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, @@ -24,6 +24,7 @@ def __init__( end_date, *, aggregation_unit, + event_types, subscriber_subset=None, sampling=None ): @@ -32,6 +33,7 @@ def __init__( self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types self.subscriber_subset = subscriber_subset self.sampling = sampling @@ -49,6 +51,7 @@ def _unsampled_query_obj(self): self.start_date, self.end_date, spatial_unit=self.aggregation_unit, + table=self.event_types, subscriber_subset=self.subscriber_subset, ) ) @@ -59,6 +62,7 @@ class UniqueLocationsSchema(AggregationUnitMixin, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["unique_locations"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) + event_types = EventTypes() subscriber_subset = SubscriberSubset() __model__ = UniqueLocationsExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/unique_subscriber_counts.py b/flowmachine/flowmachine/core/server/query_schemas/unique_subscriber_counts.py index cc766dba6a..790d6d90ab 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unique_subscriber_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unique_subscriber_counts.py @@ -15,16 +15,17 @@ __all__ = ["UniqueSubscriberCountsSchema", "UniqueSubscriberCountsExposed"] from .base_schema import BaseSchema -from .custom_fields import ISODateTime +from .custom_fields import EventTypes, ISODateTime class UniqueSubscriberCountsExposed(BaseExposedQuery): - def __init__(self, *, start_date, end_date, aggregation_unit): + def __init__(self, *, start_date, end_date, aggregation_unit, event_types): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.start_date = start_date self.end_date = end_date self.aggregation_unit = aggregation_unit + self.event_types = event_types @property def _flowmachine_query_obj(self): @@ -40,6 +41,7 @@ def _flowmachine_query_obj(self): start=self.start_date, stop=self.end_date, spatial_unit=self.aggregation_unit, + table=self.event_types, ) ) @@ -49,5 +51,6 @@ class UniqueSubscriberCountsSchema(AggregationUnitMixin, BaseSchema): query_kind = fields.String(validate=OneOf(["unique_subscriber_counts"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) + event_types = EventTypes() __model__ = UniqueSubscriberCountsExposed From 52bd2299c6c9d32331bfae84d6c110fc5ddd77dc Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 09:35:43 +0100 Subject: [PATCH 02/14] Set event_types=None if missing --- .../flowmachine/core/server/query_schemas/custom_fields.py | 7 +++++-- .../core/server/query_schemas/meaningful_locations.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/flowmachine/flowmachine/core/server/query_schemas/custom_fields.py b/flowmachine/flowmachine/core/server/query_schemas/custom_fields.py index ed4a12e7b7..7bc1c1bb0e 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/custom_fields.py +++ b/flowmachine/flowmachine/core/server/query_schemas/custom_fields.py @@ -35,7 +35,9 @@ class EventTypes(fields.List): When deserialised, will be deduped, and prefixed with "events." """ - def __init__(self, required=False, validate=None, **kwargs): + def __init__( + self, required=False, validate=None, allow_none=True, missing=None, **kwargs + ): if validate is not None: raise ValueError( "The EventTypes field provides its own validation " @@ -46,7 +48,8 @@ def __init__(self, required=False, validate=None, **kwargs): fields.String(validate=OneOf(["calls", "sms", "mds", "topups"])), required=required, validate=Length(min=1), - allow_none=True, + allow_none=allow_none, + missing=missing, **kwargs, ) diff --git a/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py b/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py index 9ef46a2940..2d0578c36d 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py +++ b/flowmachine/flowmachine/core/server/query_schemas/meaningful_locations.py @@ -54,13 +54,13 @@ def __init__( start_date: str, end_date: str, aggregation_unit: AnySpatialUnit, - event_types: Optional[Union[str, List[str]]], label: str, labels: Dict[str, Dict[str, dict]], tower_day_of_week_scores: Dict[str, float], tower_hour_of_day_scores: List[float], tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, + event_types: Optional[Union[str, List[str]]], subscriber_subset: Union[dict, None] = None, ): # Note: all input parameters need to be defined as attributes on `self` @@ -115,7 +115,6 @@ def __init__( start_date: str, end_date: str, aggregation_unit: AnySpatialUnit, - event_types: Optional[Union[str, List[str]]], label_a: str, label_b: str, labels: Dict[str, Dict[str, dict]], @@ -123,6 +122,7 @@ def __init__( tower_hour_of_day_scores: List[float], tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, + event_types: Optional[Union[str, List[str]]], subscriber_subset: Union[dict, None] = None, ): # Note: all input parameters need to be defined as attributes on `self` @@ -183,13 +183,13 @@ def __init__( start_date_b: str, end_date_b: str, aggregation_unit: AnySpatialUnit, - event_types: Optional[Union[str, List[str]]], label: str, labels: Dict[str, Dict[str, dict]], tower_day_of_week_scores: Dict[str, float], tower_hour_of_day_scores: List[float], tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, + event_types: Optional[Union[str, List[str]]], subscriber_subset: Union[dict, None] = None, ): # Note: all input parameters need to be defined as attributes on `self` From b73dd48da289b8e43baa6d52e5d69bb0e1dc0326 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 09:37:46 +0100 Subject: [PATCH 03/14] Remove subscriber_subset parameter from modal_location schema --- .../flowmachine/core/server/query_schemas/modal_location.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flowmachine/flowmachine/core/server/query_schemas/modal_location.py b/flowmachine/flowmachine/core/server/query_schemas/modal_location.py index 07c7d6049c..d897b8f13c 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/modal_location.py +++ b/flowmachine/flowmachine/core/server/query_schemas/modal_location.py @@ -6,7 +6,6 @@ from marshmallow.validate import OneOf, Length from marshmallow_oneofschema import OneOfSchema -from .custom_fields import SubscriberSubset from .daily_location import DailyLocationSchema from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, @@ -20,11 +19,10 @@ class InputToModalLocationSchema(OneOfSchema): class ModalLocationExposed(BaseExposedQueryWithSampling): - def __init__(self, locations, *, subscriber_subset=None, sampling=None): + def __init__(self, locations, *, sampling=None): # Note: all input parameters need to be defined as attributes on `self` # so that marshmallow can serialise the object correctly. self.locations = locations - self.subscriber_subset = subscriber_subset self.sampling = sampling @property @@ -48,6 +46,5 @@ class ModalLocationSchema(BaseQueryWithSamplingSchema): locations = fields.Nested( InputToModalLocationSchema, many=True, validate=Length(min=1) ) - subscriber_subset = SubscriberSubset(required=False) __model__ = ModalLocationExposed From 099c67f664906fb2c364fefec12b2041c24e667b Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 10:24:19 +0100 Subject: [PATCH 04/14] Allow table=None in TotalNetworkObjects --- .../flowmachine/features/network/total_network_objects.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flowmachine/flowmachine/features/network/total_network_objects.py b/flowmachine/flowmachine/features/network/total_network_objects.py index 77d27abffb..61ae8d56c3 100644 --- a/flowmachine/flowmachine/features/network/total_network_objects.py +++ b/flowmachine/flowmachine/features/network/total_network_objects.py @@ -84,9 +84,11 @@ def __init__( get_db().max_date(table=table) if stop is None else stop ) - self.table = table.lower() - if self.table != "all" and not self.table.startswith("events"): - self.table = "events.{}".format(self.table) + self.table = table + if isinstance(self.table, str): + self.table = self.table.lower() + if self.table != "all" and not self.table.startswith("events"): + self.table = "events.{}".format(self.table) network_object.verify_criterion("is_network_object") self.network_object = network_object From 4102579486739f4ecc9dc3bdf94445aeb79b60f3 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 10:57:40 +0100 Subject: [PATCH 05/14] Fix modal location test --- integration_tests/tests/flowmachine_server_tests/test_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/tests/flowmachine_server_tests/test_server.py b/integration_tests/tests/flowmachine_server_tests/test_server.py index 05954f3193..4e4d158a96 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_server.py +++ b/integration_tests/tests/flowmachine_server_tests/test_server.py @@ -167,7 +167,6 @@ def test_run_modal_location_query(zmq_host, zmq_port): "subscriber_subset": None, }, ], - "subscriber_subset": None, }, }, "request_id": "DUMMY_ID", From 3afde62fbd6d383ff3f6bcecbea2acf35cb51f78 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 11:43:58 +0100 Subject: [PATCH 06/14] Fix mismatched query IDs in integration tests --- .../tests/flowmachine_server_tests/test_action_get_sql.py | 1 + .../tests/flowmachine_server_tests/test_action_run_query.py | 2 ++ .../tests/flowmachine_server_tests/test_get_geo_sql.py | 1 + .../tests/flowmachine_server_tests/test_helper_functions.py | 1 + integration_tests/tests/flowmachine_server_tests/test_server.py | 1 + 5 files changed, 6 insertions(+) diff --git a/integration_tests/tests/flowmachine_server_tests/test_action_get_sql.py b/integration_tests/tests/flowmachine_server_tests/test_action_get_sql.py index fa9a5b72dd..665a839f23 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_action_get_sql.py +++ b/integration_tests/tests/flowmachine_server_tests/test_action_get_sql.py @@ -42,6 +42,7 @@ def test_get_sql(zmq_port, zmq_host): date="2016-01-01", method="last", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ) ) diff --git a/integration_tests/tests/flowmachine_server_tests/test_action_run_query.py b/integration_tests/tests/flowmachine_server_tests/test_action_run_query.py index a80df92209..09459cc176 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_action_run_query.py +++ b/integration_tests/tests/flowmachine_server_tests/test_action_run_query.py @@ -83,6 +83,7 @@ def test_run_query(zmq_port, zmq_host, fm_conn, redis): date="2016-01-01", method="last", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ) ) @@ -175,6 +176,7 @@ def test_cache_content( date="2016-01-01", method="last", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ) ) diff --git a/integration_tests/tests/flowmachine_server_tests/test_get_geo_sql.py b/integration_tests/tests/flowmachine_server_tests/test_get_geo_sql.py index ff26078ea3..e27e9262f8 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_get_geo_sql.py +++ b/integration_tests/tests/flowmachine_server_tests/test_get_geo_sql.py @@ -39,6 +39,7 @@ def test_get_geo_sql(zmq_port, zmq_host): date="2016-01-01", method="last", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ) ) diff --git a/integration_tests/tests/flowmachine_server_tests/test_helper_functions.py b/integration_tests/tests/flowmachine_server_tests/test_helper_functions.py index 3103a2294c..08c4e64dcf 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_helper_functions.py +++ b/integration_tests/tests/flowmachine_server_tests/test_helper_functions.py @@ -43,6 +43,7 @@ def test_send_zmq_message_and_receive_reply(zmq_host, zmq_port): date="2016-01-01", method="last", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ) ) diff --git a/integration_tests/tests/flowmachine_server_tests/test_server.py b/integration_tests/tests/flowmachine_server_tests/test_server.py index 4e4d158a96..8814b55729 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_server.py +++ b/integration_tests/tests/flowmachine_server_tests/test_server.py @@ -124,6 +124,7 @@ def test_run_daily_location_query(zmq_host, zmq_port): date="2016-01-01", method="most-common", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ) ) From 64ab7c0aec88e8c47162a50642fe715ba546483b Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 13:15:20 +0100 Subject: [PATCH 07/14] Add 'event_types' parameter to FlowClient --- flowclient/flowclient/aggregates.py | 48 ++++++++++++++++++++++++ flowclient/flowclient/query_specs.py | 56 +++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/flowclient/flowclient/aggregates.py b/flowclient/flowclient/aggregates.py index 792babaa78..36be5c1fd7 100644 --- a/flowclient/flowclient/aggregates.py +++ b/flowclient/flowclient/aggregates.py @@ -113,6 +113,7 @@ def meaningful_locations_aggregate_spec( aggregation_unit: str, tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, + event_types: Union[None, List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -167,6 +168,8 @@ def meaningful_locations_aggregate_spec( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -197,6 +200,7 @@ def meaningful_locations_aggregate_spec( "tower_hour_of_day_scores": tower_hour_of_day_scores, "tower_cluster_radius": tower_cluster_radius, "tower_cluster_call_threshold": tower_cluster_call_threshold, + "event_types": event_types, "subscriber_subset": subscriber_subset, "mapping_table": mapping_table, "geom_table": geom_table, @@ -257,6 +261,8 @@ def meaningful_locations_aggregate(*, connection: Connection, **kwargs) -> APIQu more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -293,6 +299,7 @@ def meaningful_locations_between_label_od_matrix_spec( aggregation_unit: str, tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, + event_types: Union[None, List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -347,6 +354,8 @@ def meaningful_locations_between_label_od_matrix_spec( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -378,6 +387,7 @@ def meaningful_locations_between_label_od_matrix_spec( "tower_hour_of_day_scores": tower_hour_of_day_scores, "tower_cluster_radius": tower_cluster_radius, "tower_cluster_call_threshold": tower_cluster_call_threshold, + "event_types": event_types, "subscriber_subset": subscriber_subset, "mapping_table": mapping_table, "geom_table": geom_table, @@ -440,6 +450,8 @@ def meaningful_locations_between_label_od_matrix( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -477,6 +489,7 @@ def meaningful_locations_between_dates_od_matrix_spec( aggregation_unit: str, tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: float = 0, + event_types: Union[None, List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -536,6 +549,8 @@ def meaningful_locations_between_dates_od_matrix_spec( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -568,6 +583,7 @@ def meaningful_locations_between_dates_od_matrix_spec( "tower_hour_of_day_scores": tower_hour_of_day_scores, "tower_cluster_radius": tower_cluster_radius, "tower_cluster_call_threshold": tower_cluster_call_threshold, + "event_types": event_types, "subscriber_subset": subscriber_subset, "mapping_table": mapping_table, "geom_table": geom_table, @@ -635,6 +651,8 @@ def meaningful_locations_between_dates_od_matrix( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -718,6 +736,7 @@ def unique_subscriber_counts_spec( mapping_table: Optional[str] = None, geom_table: Optional[str] = None, geom_table_join_column: Optional[str] = None, + event_types: Union[None, List[str]] = None, ) -> dict: """ Return query spec for unique subscriber counts @@ -730,6 +749,8 @@ def unique_subscriber_counts_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" Returns ------- @@ -744,6 +765,7 @@ def unique_subscriber_counts_spec( "mapping_table": mapping_table, "geom_table": geom_table, "geom_table_join_column": geom_table_join_column, + "event_types": event_types, } @@ -762,6 +784,8 @@ def unique_subscriber_counts(*, connection: Connection, **kwargs) -> APIQuery: ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" Returns ------- @@ -780,6 +804,7 @@ def location_introversion_spec( mapping_table: Optional[str] = None, geom_table: Optional[str] = None, geom_table_join_column: Optional[str] = None, + event_types: Union[None, List[str]] = None, ) -> dict: """ Return query spec for location introversion @@ -794,6 +819,8 @@ def location_introversion_spec( Unit of aggregation, e.g. "admin3" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts can be one of "in", "out" or "both" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" Returns ------- @@ -809,6 +836,7 @@ def location_introversion_spec( "mapping_table": mapping_table, "geom_table": geom_table, "geom_table_join_column": geom_table_join_column, + "event_types": event_types, } @@ -829,6 +857,8 @@ def location_introversion(*, connection: Connection, **kwargs) -> APIQuery: Unit of aggregation, e.g. "admin3" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts can be one of "in", "out" or "both" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" Returns ------- @@ -847,6 +877,7 @@ def total_network_objects_spec( mapping_table: Optional[str] = None, geom_table: Optional[str] = None, geom_table_join_column: Optional[str] = None, + event_types: Union[None, List[str]] = None, ) -> dict: """ Return query spec for total network objects @@ -861,6 +892,8 @@ def total_network_objects_spec( Unit of aggregation, e.g. "admin3" total_by : {"second", "minute", "hour", "day", "month", "year"} Time period to bucket by one of "second", "minute", "hour", "day", "month" or "year" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" Returns ------- @@ -876,6 +909,7 @@ def total_network_objects_spec( "mapping_table": mapping_table, "geom_table": geom_table, "geom_table_join_column": geom_table_join_column, + "event_types": event_types, } @@ -896,6 +930,8 @@ def total_network_objects(*, connection: Connection, **kwargs) -> APIQuery: Unit of aggregation, e.g. "admin3" total_by : {"second", "minute", "hour", "day", "month", "year"} Time period to bucket by one of "second", "minute", "hour", "day", "month" or "year" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" Returns ------- @@ -1003,6 +1039,7 @@ def consecutive_trips_od_matrix_spec( start_date: str, end_date: str, aggregation_unit: str, + event_types: Union[None, List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -1017,6 +1054,8 @@ def consecutive_trips_od_matrix_spec( ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -1033,6 +1072,7 @@ def consecutive_trips_od_matrix_spec( start_date=start_date, end_date=end_date, aggregation_unit=aggregation_unit, + event_types=event_types, subscriber_subset=subscriber_subset, mapping_table=mapping_table, geom_table=geom_table, @@ -1053,6 +1093,8 @@ def consecutive_trips_od_matrix(*, connection: Connection, **kwargs) -> APIQuery ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -1073,6 +1115,7 @@ def trips_od_matrix_spec( start_date: str, end_date: str, aggregation_unit: str, + event_types: Union[None, List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -1087,6 +1130,8 @@ def trips_od_matrix_spec( ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -1103,6 +1148,7 @@ def trips_od_matrix_spec( start_date=start_date, end_date=end_date, aggregation_unit=aggregation_unit, + event_types=event_types, subscriber_subset=subscriber_subset, mapping_table=mapping_table, geom_table=geom_table, @@ -1123,6 +1169,8 @@ def trips_od_matrix(*, connection: Connection, **kwargs) -> APIQuery: ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : None or list of {"calls", "sms", "mds"}, default None + Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a diff --git a/flowclient/flowclient/query_specs.py b/flowclient/flowclient/query_specs.py index de16076d2a..2fd5c7ae8c 100644 --- a/flowclient/flowclient/query_specs.py +++ b/flowclient/flowclient/query_specs.py @@ -12,6 +12,7 @@ def unique_locations_spec( start_date: str, end_date: str, aggregation_unit: str, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -27,6 +28,9 @@ def unique_locations_spec( ISO format dates between which to get unique locations, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None Subset of subscribers to retrieve daily locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -43,6 +47,7 @@ def unique_locations_spec( start_date=start_date, end_date=end_date, aggregation_unit=aggregation_unit, + event_types=event_types, subscriber_subset=subscriber_subset, mapping_table=mapping_table, geom_table=geom_table, @@ -55,6 +60,7 @@ def daily_location_spec( date: str, aggregation_unit: str, method: str, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -72,6 +78,9 @@ def daily_location_spec( Unit of aggregation, e.g. "admin3" method : str Method to use for daily location, one of 'last' or 'most-common' + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None Subset of subscribers to retrieve daily locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -88,6 +97,7 @@ def daily_location_spec( "date": date, "aggregation_unit": aggregation_unit, "method": method, + "event_types": event_types, "subscriber_subset": subscriber_subset, "mapping_table": mapping_table, "geom_table": geom_table, @@ -126,6 +136,7 @@ def modal_location_from_dates_spec( end_date: str, aggregation_unit: str, method: str, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -145,6 +156,9 @@ def modal_location_from_dates_spec( Unit of aggregation, e.g. "admin3" method : str Method to use for daily locations, one of 'last' or 'most-common' + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -165,6 +179,7 @@ def modal_location_from_dates_spec( date=date, aggregation_unit=aggregation_unit, method=method, + event_types=event_types, subscriber_subset=subscriber_subset, mapping_table=mapping_table, geom_table=geom_table, @@ -176,7 +191,11 @@ def modal_location_from_dates_spec( def radius_of_gyration_spec( - *, start_date: str, end_date: str, subscriber_subset: Union[dict, None] = None + *, + start_date: str, + end_date: str, + event_types: Optional[List[str]] = None, + subscriber_subset: Union[dict, None] = None, ) -> dict: """ Return query spec for radius of gyration @@ -187,6 +206,9 @@ def radius_of_gyration_spec( ISO format date of the first day of the count, e.g. "2016-01-01" end_date : str ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -201,6 +223,7 @@ def radius_of_gyration_spec( "query_kind": "radius_of_gyration", "start_date": start_date, "end_date": end_date, + "event_types": event_types, "subscriber_subset": subscriber_subset, } @@ -210,6 +233,7 @@ def unique_location_counts_spec( start_date: str, end_date: str, aggregation_unit: str, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -226,6 +250,9 @@ def unique_location_counts_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" aggregation_unit : str Unit of aggregation, e.g. "admin3" + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -241,6 +268,7 @@ def unique_location_counts_spec( "start_date": start_date, "end_date": end_date, "aggregation_unit": aggregation_unit, + "event_types": event_types, "subscriber_subset": subscriber_subset, "mapping_table": mapping_table, "geom_table": geom_table, @@ -290,6 +318,7 @@ def subscriber_degree_spec( start: str, stop: str, direction: str = "both", + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, ) -> dict: """ @@ -303,6 +332,9 @@ def subscriber_degree_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts. Can be one of "in", "out" or "both". + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -318,6 +350,7 @@ def subscriber_degree_spec( "start": start, "stop": stop, "direction": direction, + "event_types": event_types, "subscriber_subset": subscriber_subset, } @@ -407,6 +440,7 @@ def displacement_spec( stop: str, statistic: str, reference_location: Dict[str, str], + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, ) -> dict: """ @@ -422,6 +456,9 @@ def displacement_spec( Statistic type one of "avg", "max", "min", "median", "mode", "stddev" or "variance". reference_location: + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -438,6 +475,7 @@ def displacement_spec( "stop": stop, "statistic": statistic, "reference_location": reference_location, + "event_types": event_types, "subscriber_subset": subscriber_subset, } @@ -447,6 +485,7 @@ def pareto_interactions_spec( start: str, stop: str, proportion: float, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, ) -> dict: """ @@ -460,6 +499,9 @@ def pareto_interactions_spec( ISO format date of the day _after_ the final date of the time interval to be considered, e.g. "2016-01-08" proportion : float proportion to track below + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in result. Must be None (= all subscribers) or a dictionary with the specification of a @@ -475,6 +517,7 @@ def pareto_interactions_spec( "start": start, "stop": stop, "proportion": proportion, + "event_types": event_types, "subscriber_subset": subscriber_subset, } @@ -484,6 +527,7 @@ def nocturnal_events_spec( start: str, stop: str, hours: Tuple[int, int], + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, ) -> dict: """ @@ -497,7 +541,9 @@ def nocturnal_events_spec( ISO format date of the day _after_ the final date for which to count nocturnal events, e.g. "2016-01-08" hours: tuple(int,int) Tuple defining beginning and end of night - + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -515,6 +561,7 @@ def nocturnal_events_spec( "stop": stop, "night_start_hour": hours[0], "night_end_hour": hours[1], + "event_types": event_types, "subscriber_subset": subscriber_subset, } @@ -525,6 +572,7 @@ def handset_spec( end_date: str, characteristic: str = "hnd_type", method: str = "last", + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, ) -> dict: """ @@ -540,6 +588,9 @@ def handset_spec( The required handset characteristic. method: {"last", "most-common"}, default "last" Method for choosing a handset to associate with subscriber. + event_types : list of str, optional + The event types to include in the count (for example: ["calls", "sms"]). + If None, include all event types in the count. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -556,6 +607,7 @@ def handset_spec( "end_date": end_date, "characteristic": characteristic, "method": method, + "event_types": event_types, "subscriber_subset": subscriber_subset, } From ebcd6985b28e0a5f3324eb0f7f9af567c98df849 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 14:30:53 +0100 Subject: [PATCH 08/14] Fix another test --- integration_tests/tests/flowmachine_server_tests/test_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_tests/tests/flowmachine_server_tests/test_server.py b/integration_tests/tests/flowmachine_server_tests/test_server.py index 8814b55729..4b256a7335 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_server.py +++ b/integration_tests/tests/flowmachine_server_tests/test_server.py @@ -181,12 +181,14 @@ def test_run_modal_location_query(zmq_host, zmq_port): date="2016-01-01", method="most-common", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ), daily_location( date="2016-01-02", method="most-common", spatial_unit=make_spatial_unit("admin", level=3), + table=None, subscriber_subset=None, ), ) From 9047a58907d99c8675a83cde6b9246bcd0655e05 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 17:51:12 +0100 Subject: [PATCH 09/14] Add tests for 'event_types' parameter --- .../tests/test_query_object_construction.py | 4 + .../tests/query_tests/test_queries.py | 116 +++++++++++++++++- 2 files changed, 115 insertions(+), 5 deletions(-) diff --git a/flowmachine/tests/test_query_object_construction.py b/flowmachine/tests/test_query_object_construction.py index 6990062af4..73e9db9e51 100644 --- a/flowmachine/tests/test_query_object_construction.py +++ b/flowmachine/tests/test_query_object_construction.py @@ -21,6 +21,7 @@ def test_construct_query(diff_reporter): "date": "2016-01-01", "aggregation_unit": "admin3", "method": "last", + "event_types": ["calls", "sms"], "subscriber_subset": None, "sampling": { "sampling_method": "bernoulli", @@ -48,6 +49,7 @@ def test_construct_query(diff_reporter): "date": "2016-01-01", "aggregation_unit": "admin3", "method": "last", + "event_types": None, "subscriber_subset": None, "sampling": None, }, @@ -202,6 +204,7 @@ def test_construct_query(diff_reporter): }, "tower_cluster_radius": 1.0, "tower_cluster_call_threshold": 0, + "event_types": None, "subscriber_subset": None, }, { @@ -263,6 +266,7 @@ def test_construct_query(diff_reporter): }, "tower_cluster_radius": 1.0, "tower_cluster_call_threshold": 2, + "event_types": ["calls", "sms"], "subscriber_subset": None, }, ] diff --git a/integration_tests/tests/query_tests/test_queries.py b/integration_tests/tests/query_tests/test_queries.py index b7e4881f71..0f1b7b58e4 100644 --- a/integration_tests/tests/query_tests/test_queries.py +++ b/integration_tests/tests/query_tests/test_queries.py @@ -28,6 +28,7 @@ date="2016-01-01", aggregation_unit="admin3", method="most-common", + event_types=["calls", "sms"], subscriber_subset=None, ), ), @@ -37,6 +38,13 @@ end_date="2016-01-02", aggregation_unit="admin3", ), + partial( + flowclient.unique_subscriber_counts, + start_date="2016-01-01", + end_date="2016-01-02", + aggregation_unit="admin3", + event_types=["calls", "sms"], + ), partial( flowclient.total_network_objects, start_date="2016-01-01", @@ -49,6 +57,7 @@ end_date="2016-01-02", aggregation_unit="admin3", total_by="day", + event_types=["calls", "sms"], ), partial( flowclient.location_introversion, @@ -62,6 +71,7 @@ end_date="2016-01-02", aggregation_unit="admin3", direction="in", + event_types=["calls", "sms"], ), partial( flowclient.aggregate_network_objects, @@ -85,10 +95,12 @@ partial( flowclient.joined_spatial_aggregate, locations=flowclient.daily_location_spec( - date="2016-01-01", aggregation_unit="admin3", method="last" + date="2016-01-01", aggregation_unit="admin3", method="last", ), metric=flowclient.radius_of_gyration_spec( - start_date="2016-01-01", end_date="2016-01-02" + start_date="2016-01-01", + end_date="2016-01-02", + event_types=["calls", "sms"], ), ), partial( @@ -97,7 +109,10 @@ date="2016-01-01", aggregation_unit="admin3", method="last" ), metric=flowclient.nocturnal_events_spec( - start="2016-01-01", stop="2016-01-02", hours=(20, 4) + start="2016-01-01", + stop="2016-01-02", + hours=(20, 4), + event_types=["calls", "sms"], ), ), partial( @@ -106,7 +121,10 @@ date="2016-01-01", aggregation_unit="admin3", method="last" ), metric=flowclient.subscriber_degree_spec( - start="2016-01-01", stop="2016-01-02", direction="both" + start="2016-01-01", + stop="2016-01-02", + direction="both", + event_types=["calls", "sms"], ), ), partial( @@ -118,6 +136,7 @@ start_date="2016-01-01", end_date="2016-01-02", aggregation_unit="admin3", + event_types=["calls", "sms"], ), ), partial( @@ -153,6 +172,7 @@ reference_location=flowclient.daily_location_spec( date="2016-01-01", aggregation_unit="lon-lat", method="last" ), + event_types=["calls", "sms"], ), ), partial( @@ -161,7 +181,19 @@ date="2016-01-01", aggregation_unit="admin3", method="last" ), metric=flowclient.pareto_interactions_spec( - start="2016-01-01", stop="2016-01-02", proportion="0.8" + start="2016-01-01", + stop="2016-01-02", + proportion="0.8", + event_types=["calls", "sms"], + ), + ), + partial( + flowclient.spatial_aggregate, + locations=flowclient.modal_location_from_dates_spec( + start_date="2016-01-01", + end_date="2016-01-03", + aggregation_unit="admin3", + method="last", ), ), partial( @@ -171,6 +203,7 @@ end_date="2016-01-03", aggregation_unit="admin3", method="last", + event_types=["calls", "sms"], ), ), partial( @@ -206,6 +239,7 @@ start_date="2016-01-01", end_date="2016-01-04", aggregation_unit="admin3", + event_types=["calls", "sms"], ), ), partial( @@ -300,6 +334,61 @@ ), ), ), + partial( + flowclient.meaningful_locations_aggregate, + start_date="2016-01-01", + end_date="2016-01-02", + aggregation_unit="admin1", + label="unknown", + tower_hour_of_day_scores=[ + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + ], + tower_day_of_week_scores=dict( + monday=1, + tuesday=1, + wednesday=1, + thursday=0, + friday=-1, + saturday=-1, + sunday=-1, + ), + labels=dict( + evening=dict( + type="Polygon", + coordinates=[ + [[1e-06, -0.5], [1e-06, -1.1], [1.1, -1.1], [1.1, -0.5]] + ], + ), + day=dict( + type="Polygon", + coordinates=[[[-1.1, -0.5], [-1.1, 0.5], [-1e-06, 0.5], [0, -0.5]]], + ), + ), + event_types=["calls", "sms"], + ), partial( flowclient.meaningful_locations_between_label_od_matrix, start_date="2016-01-01", @@ -354,6 +443,7 @@ coordinates=[[[-1.1, -0.5], [-1.1, 0.5], [-1e-06, 0.5], [0, -0.5]]], ), ), + event_types=["calls", "sms"], ), partial( flowclient.location_event_counts, @@ -430,6 +520,7 @@ coordinates=[[[-1.1, -0.5], [-1.1, 0.5], [-1e-06, 0.5], [0, -0.5]]], ), ), + event_types=["calls", "sms"], ), partial( flowclient.joined_spatial_aggregate, @@ -472,6 +563,7 @@ end_date="2016-01-02", characteristic="brand", method="last", + event_types=["calls", "sms"], ), method="distr", ), @@ -538,11 +630,25 @@ end_date="2016-01-03", aggregation_unit="admin3", ), + partial( + flowclient.consecutive_trips_od_matrix, + start_date="2016-01-01", + end_date="2016-01-03", + aggregation_unit="admin3", + event_types=["calls", "sms"], + ), + partial( + flowclient.trips_od_matrix, + start_date="2016-01-01", + end_date="2016-01-03", + aggregation_unit="admin3", + ), partial( flowclient.trips_od_matrix, start_date="2016-01-01", end_date="2016-01-03", aggregation_unit="admin3", + event_types=["calls", "sms"], ), ], ids=lambda val: val.func.__name__, From ab88f5463b10b89736055170755da8dc8b04b63a Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 17:51:24 +0100 Subject: [PATCH 10/14] Re-approve API spec --- ..._of_flowmachine_query_schemas.approved.txt | 264 +++++++++++++++++- 1 file changed, 257 insertions(+), 7 deletions(-) diff --git a/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt b/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt index 8ab6ba39d5..98b28d8986 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt +++ b/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt @@ -126,6 +126,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -223,6 +238,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -273,6 +303,21 @@ }, "Displacement": { "properties": { + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "query_kind": { "enum": [ "displacement" @@ -381,6 +426,7 @@ "type": "string" }, "event_types": { + "default": null, "items": { "enum": [ "calls", @@ -596,6 +642,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "method": { "enum": [ "last", @@ -895,6 +956,7 @@ "type": "string" }, "event_types": { + "default": null, "items": { "enum": [ "calls", @@ -979,6 +1041,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1025,6 +1102,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1123,6 +1215,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1222,6 +1329,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1325,19 +1447,27 @@ } ], "nullable": true - }, - "subscriber_subset": { - "enum": [ - null - ], - "nullable": true, - "type": "string" } }, "type": "object" }, "NocturnalEvents": { "properties": { + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "night_end_hour": { "format": "int32", "maximum": 23, @@ -1388,6 +1518,21 @@ }, "ParetoInteractions": { "properties": { + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "proportion": { "format": "float", "maximum": 1.0, @@ -1437,6 +1582,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "query_kind": { "enum": [ "radius_of_gyration" @@ -1571,6 +1731,21 @@ ], "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "query_kind": { "enum": [ "subscriber_degree" @@ -1760,6 +1935,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1818,6 +2008,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1871,6 +2076,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1924,6 +2144,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1985,6 +2220,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" From dabf903fd338764182d6a5cff201ef123e2ca70b Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 18:05:07 +0100 Subject: [PATCH 11/14] Re-approve another --- ...t_generated_openapi_json_spec.approved.txt | 264 +++++++++++++++++- 1 file changed, 257 insertions(+), 7 deletions(-) diff --git a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt index 097e30e775..dd556421ae 100644 --- a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt +++ b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt @@ -129,6 +129,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -228,6 +243,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -279,6 +309,21 @@ }, "Displacement": { "properties": { + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "query_kind": { "enum": [ "displacement" @@ -389,6 +434,7 @@ "type": "string" }, "event_types": { + "default": null, "items": { "enum": [ "calls", @@ -607,6 +653,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "method": { "enum": [ "last", @@ -909,6 +970,7 @@ "type": "string" }, "event_types": { + "default": null, "items": { "enum": [ "calls", @@ -994,6 +1056,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1041,6 +1118,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1140,6 +1232,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1240,6 +1347,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1344,19 +1466,27 @@ } ], "nullable": true - }, - "subscriber_subset": { - "enum": [ - null - ], - "nullable": true, - "type": "string" } }, "type": "object" }, "NocturnalEvents": { "properties": { + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "night_end_hour": { "format": "int32", "maximum": 23, @@ -1408,6 +1538,21 @@ }, "ParetoInteractions": { "properties": { + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "proportion": { "format": "float", "maximum": 1.0, @@ -1458,6 +1603,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "query_kind": { "enum": [ "radius_of_gyration" @@ -1594,6 +1754,21 @@ ], "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "query_kind": { "enum": [ "subscriber_degree" @@ -1786,6 +1961,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1845,6 +2035,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1899,6 +2104,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -1953,6 +2173,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" @@ -2015,6 +2250,21 @@ "format": "date-time", "type": "string" }, + "event_types": { + "default": null, + "items": { + "enum": [ + "calls", + "mds", + "sms", + "topups" + ], + "type": "string" + }, + "minItems": 1, + "nullable": true, + "type": "array" + }, "geom_table": { "nullable": true, "type": "string" From d08133b45e2452c7d29d0eb960f469f56c468c6e Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 18:10:31 +0100 Subject: [PATCH 12/14] Approve new query IDs --- ...truction.test_construct_query.approved.txt | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/flowmachine/tests/test_query_object_construction.test_construct_query.approved.txt b/flowmachine/tests/test_query_object_construction.test_construct_query.approved.txt index 0457348f4d..f3631180e5 100644 --- a/flowmachine/tests/test_query_object_construction.test_construct_query.approved.txt +++ b/flowmachine/tests/test_query_object_construction.test_construct_query.approved.txt @@ -1,11 +1,15 @@ { - "6353b1246a73d0f70de07ae191a95c8b": { + "5e5099f1f762584cc8d6ed360695443c": { "query_kind": "spatial_aggregate", "locations": { "query_kind": "daily_location", "date": "2016-01-01", "aggregation_unit": "admin3", "method": "last", + "event_types": [ + "calls", + "sms" + ], "subscriber_subset": null, "sampling": { "sampling_method": "bernoulli", @@ -16,13 +20,14 @@ } } }, - "aa18ff4cc8cd4baf0c3d5c72a75a3cd9": { + "7ed1e4e947727ca17e099e172774e908": { "query_kind": "spatial_aggregate", "locations": { "query_kind": "daily_location", "date": "2016-01-01", "aggregation_unit": "admin3", "method": "last", + "event_types": null, "subscriber_subset": null, "sampling": null } @@ -37,7 +42,7 @@ "event_types": null, "subscriber_subset": null }, - "b155478a246bca7fa30094c51cce4faf": { + "4d6b6efb7090b7e0ddf69620bc6cfd40": { "query_kind": "spatial_aggregate", "locations": { "query_kind": "modal_location", @@ -63,7 +68,7 @@ "query_kind": "geography", "aggregation_unit": "admin3" }, - "e3093957d5792d9fca6797eee24758a5": { + "ab27899043f9b74e80b0cc54fe7b75d2": { "query_kind": "meaningful_locations_aggregate", "aggregation_unit": "admin1", "start_date": "2016-01-01", @@ -156,7 +161,7 @@ "tower_cluster_call_threshold": 0, "subscriber_subset": null }, - "82b0a2a1b274aade7de018195eaec86c": { + "496a236d337dce47b9a0419c2004163a": { "query_kind": "meaningful_locations_between_label_od_matrix", "aggregation_unit": "admin1", "start_date": "2016-01-01", @@ -248,9 +253,10 @@ }, "tower_cluster_radius": 1.0, "tower_cluster_call_threshold": 0, + "event_types": null, "subscriber_subset": null }, - "9c255f14595818421f0b0a72bc3497b8": { + "a2a567fc5936fde732adb6dc241cb63c": { "query_kind": "meaningful_locations_between_dates_od_matrix", "aggregation_unit": "admin1", "start_date_a": "2016-01-01", @@ -343,6 +349,10 @@ }, "tower_cluster_radius": 1.0, "tower_cluster_call_threshold": 2, + "event_types": [ + "calls", + "sms" + ], "subscriber_subset": null } } \ No newline at end of file From 19765813619300546f9daa5baee2310e224b38e1 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Tue, 2 Jun 2020 18:19:51 +0100 Subject: [PATCH 13/14] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d64ac21304..0be6d7b85b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added +- Queries run through FlowAPI can now be run on only a subset of the available CDR types, by supplying an `event_types` parameter. [#2631](https://github.com/Flowminder/FlowKit/issues/2631) ### Changed From 2bac3993026f4d1d0a46de814031e6d2afc277c2 Mon Sep 17 00:00:00 2001 From: James Harrison Date: Wed, 3 Jun 2020 12:10:23 +0100 Subject: [PATCH 14/14] Consistent type hints and docstrings --- flowclient/flowclient/aggregates.py | 108 ++++++++++++++++----------- flowclient/flowclient/query_specs.py | 71 +++++++++--------- 2 files changed, 99 insertions(+), 80 deletions(-) diff --git a/flowclient/flowclient/aggregates.py b/flowclient/flowclient/aggregates.py index 36be5c1fd7..16f73b05a5 100644 --- a/flowclient/flowclient/aggregates.py +++ b/flowclient/flowclient/aggregates.py @@ -18,7 +18,7 @@ def location_event_counts_spec( aggregation_unit: str, count_interval: str, direction: str = "both", - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -40,8 +40,9 @@ def location_event_counts_spec( Can be one of "day", "hour" or "minute". direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts. Can be one of "in", "out" or "both". - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -87,8 +88,9 @@ def location_event_counts(*, connection: Connection, **kwargs) -> APIQuery: Can be one of "day", "hour" or "minute". direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts. Can be one of "in", "out" or "both". - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -113,7 +115,7 @@ def meaningful_locations_aggregate_spec( aggregation_unit: str, tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -168,8 +170,9 @@ def meaningful_locations_aggregate_spec( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -261,8 +264,9 @@ def meaningful_locations_aggregate(*, connection: Connection, **kwargs) -> APIQu more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -299,7 +303,7 @@ def meaningful_locations_between_label_od_matrix_spec( aggregation_unit: str, tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: int = 0, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -354,8 +358,9 @@ def meaningful_locations_between_label_od_matrix_spec( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -450,8 +455,9 @@ def meaningful_locations_between_label_od_matrix( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -489,7 +495,7 @@ def meaningful_locations_between_dates_od_matrix_spec( aggregation_unit: str, tower_cluster_radius: float = 1.0, tower_cluster_call_threshold: float = 0, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -549,8 +555,9 @@ def meaningful_locations_between_dates_od_matrix_spec( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -651,8 +658,9 @@ def meaningful_locations_between_dates_od_matrix( more towers, and fewer clusters. tower_cluster_call_threshold : int Exclude towers from a subscriber's clusters if they have been used on less than this number of days. - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -736,7 +744,7 @@ def unique_subscriber_counts_spec( mapping_table: Optional[str] = None, geom_table: Optional[str] = None, geom_table_join_column: Optional[str] = None, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, ) -> dict: """ Return query spec for unique subscriber counts @@ -749,8 +757,9 @@ def unique_subscriber_counts_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. Returns ------- @@ -784,8 +793,9 @@ def unique_subscriber_counts(*, connection: Connection, **kwargs) -> APIQuery: ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. Returns ------- @@ -804,7 +814,7 @@ def location_introversion_spec( mapping_table: Optional[str] = None, geom_table: Optional[str] = None, geom_table_join_column: Optional[str] = None, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, ) -> dict: """ Return query spec for location introversion @@ -819,8 +829,9 @@ def location_introversion_spec( Unit of aggregation, e.g. "admin3" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts can be one of "in", "out" or "both" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. Returns ------- @@ -857,8 +868,9 @@ def location_introversion(*, connection: Connection, **kwargs) -> APIQuery: Unit of aggregation, e.g. "admin3" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts can be one of "in", "out" or "both" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. Returns ------- @@ -877,7 +889,7 @@ def total_network_objects_spec( mapping_table: Optional[str] = None, geom_table: Optional[str] = None, geom_table_join_column: Optional[str] = None, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, ) -> dict: """ Return query spec for total network objects @@ -892,8 +904,9 @@ def total_network_objects_spec( Unit of aggregation, e.g. "admin3" total_by : {"second", "minute", "hour", "day", "month", "year"} Time period to bucket by one of "second", "minute", "hour", "day", "month" or "year" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. Returns ------- @@ -930,8 +943,9 @@ def total_network_objects(*, connection: Connection, **kwargs) -> APIQuery: Unit of aggregation, e.g. "admin3" total_by : {"second", "minute", "hour", "day", "month", "year"} Time period to bucket by one of "second", "minute", "hour", "day", "month" or "year" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. Returns ------- @@ -1039,7 +1053,7 @@ def consecutive_trips_od_matrix_spec( start_date: str, end_date: str, aggregation_unit: str, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -1054,8 +1068,9 @@ def consecutive_trips_od_matrix_spec( ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -1093,8 +1108,9 @@ def consecutive_trips_od_matrix(*, connection: Connection, **kwargs) -> APIQuery ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -1115,7 +1131,7 @@ def trips_od_matrix_spec( start_date: str, end_date: str, aggregation_unit: str, - event_types: Union[None, List[str]] = None, + event_types: Optional[List[str]] = None, subscriber_subset: Union[dict, None] = None, mapping_table: Optional[str] = None, geom_table: Optional[str] = None, @@ -1130,8 +1146,9 @@ def trips_od_matrix_spec( ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -1169,8 +1186,9 @@ def trips_od_matrix(*, connection: Connection, **kwargs) -> APIQuery: ISO format dates between which to find trips, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : None or list of {"calls", "sms", "mds"}, default None - Optionally, include only a subset of events. Can be one of "calls", "sms" or "mds" + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve trips for. Must be None (= all subscribers) or a dictionary with the specification of a diff --git a/flowclient/flowclient/query_specs.py b/flowclient/flowclient/query_specs.py index 2fd5c7ae8c..daf6615215 100644 --- a/flowclient/flowclient/query_specs.py +++ b/flowclient/flowclient/query_specs.py @@ -28,9 +28,9 @@ def unique_locations_spec( ISO format dates between which to get unique locations, e.g. "2016-01-01" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve daily locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -78,9 +78,9 @@ def daily_location_spec( Unit of aggregation, e.g. "admin3" method : str Method to use for daily location, one of 'last' or 'most-common' - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve daily locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -156,9 +156,9 @@ def modal_location_from_dates_spec( Unit of aggregation, e.g. "admin3" method : str Method to use for daily locations, one of 'last' or 'most-common' - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None Subset of subscribers to retrieve modal locations for. Must be None (= all subscribers) or a dictionary with the specification of a @@ -206,9 +206,9 @@ def radius_of_gyration_spec( ISO format date of the first day of the count, e.g. "2016-01-01" end_date : str ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -250,9 +250,9 @@ def unique_location_counts_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" aggregation_unit : str Unit of aggregation, e.g. "admin3" - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -332,9 +332,9 @@ def subscriber_degree_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts. Can be one of "in", "out" or "both". - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -411,9 +411,9 @@ def event_count_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" direction : {"in", "out", "both"}, default "both" Optionally, include only ingoing or outbound calls/texts. Can be one of "in", "out" or "both". - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -454,11 +454,12 @@ def displacement_spec( ISO format date of the day _after_ the final date of the count, e.g. "2016-01-08" statistic : {"avg", "max", "min", "median", "mode", "stddev", "variance"} Statistic type one of "avg", "max", "min", "median", "mode", "stddev" or "variance". - reference_location: - - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + reference_location : dict + Query specification for the locations (daily or modal location) from which to + calculate displacement. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -499,9 +500,9 @@ def pareto_interactions_spec( ISO format date of the day _after_ the final date of the time interval to be considered, e.g. "2016-01-08" proportion : float proportion to track below - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in result. Must be None (= all subscribers) or a dictionary with the specification of a @@ -541,9 +542,9 @@ def nocturnal_events_spec( ISO format date of the day _after_ the final date for which to count nocturnal events, e.g. "2016-01-08" hours: tuple(int,int) Tuple defining beginning and end of night - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a @@ -588,9 +589,9 @@ def handset_spec( The required handset characteristic. method: {"last", "most-common"}, default "last" Method for choosing a handset to associate with subscriber. - event_types : list of str, optional - The event types to include in the count (for example: ["calls", "sms"]). - If None, include all event types in the count. + event_types : list of {"calls", "sms", "mds", "topups"}, optional + Optionally, include only a subset of event types (for example: ["calls", "sms"]). + If None, include all event types in the query. subscriber_subset : dict or None, default None Subset of subscribers to include in event counts. Must be None (= all subscribers) or a dictionary with the specification of a