From 0b96e148036ca185b120584fdfbbed3f5d86cf3f Mon Sep 17 00:00:00 2001 From: robcarver17 Date: Wed, 28 Oct 2020 14:37:46 +0000 Subject: [PATCH] changed data blob inputs v0.51.0 --- DONE_TO_DO.md | 5 + README.md | 4 +- sysproduction/backup_arctic_to_csv.py | 36 +-- sysproduction/data/broker.py | 14 +- sysproduction/data/capital.py | 3 +- sysproduction/data/contracts.py | 13 +- sysproduction/data/controls.py | 18 +- sysproduction/data/currency_data.py | 9 +- sysproduction/data/get_data.py | 361 ++++++++++---------------- sysproduction/data/instruments.py | 7 +- sysproduction/data/logs.py | 5 +- sysproduction/data/orders.py | 13 +- sysproduction/data/positions.py | 13 +- sysproduction/data/prices.py | 26 +- sysproduction/data/sim_data.py | 6 +- sysproduction/data/volumes.py | 7 +- sysproduction/diagnostic/emailing.py | 4 +- 17 files changed, 259 insertions(+), 285 deletions(-) diff --git a/DONE_TO_DO.md b/DONE_TO_DO.md index cf706969f6..25608c1f3f 100644 --- a/DONE_TO_DO.md +++ b/DONE_TO_DO.md @@ -1,5 +1,10 @@ # Release notes +## Version 0.51.0 + +Essentially 'finished' production.md (in as much as anything can be finished...) +Changed data Blobs so now take lists of objects rather than str, easier to see dependencies + ## Version 0.50.0 (Done loads of work but forgotten to update the version number or this file. So let's reward ourselves with a 0.20 version bump. The following list is almost certainly incomplete...) diff --git a/README.md b/README.md index 11e2e03795..1a04a57da4 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ Rob Carver [https://qoppac.blogspot.com/p/pysystemtrade.html](https://qoppac.blogspot.com/p/pysystemtrade.html) -Version 0.50.0 +Version 0.51.0 -20201023 +20201028 ## Release notes diff --git a/sysproduction/backup_arctic_to_csv.py b/sysproduction/backup_arctic_to_csv.py index 9e7dc99952..2524cddf92 100644 --- a/sysproduction/backup_arctic_to_csv.py +++ b/sysproduction/backup_arctic_to_csv.py @@ -7,6 +7,20 @@ from sysproduction.data.strategies import get_list_of_strategies import os +from sysdata.csv.csv_futures_contracts import csvFuturesContractData +from sysdata.csv.csv_adjusted_prices import csvFuturesAdjustedPricesData +from sysdata.csv.csv_futures_contract_prices import csvFuturesContractPriceData +from sysdata.csv.csv_multiple_prices import csvFuturesMultiplePricesData +from sysdata.csv.csv_spot_fx import csvFxPricesData +from sysdata.csv.csv_contract_position_data import csvContractPositionData +from sysdata.csv.csv_strategy_position_data import csvStrategyPositionData +from sysdata.csv.csv_historic_orders import csvStrategyHistoricOrdersData, csvContractHistoricOrdersData, csvBrokerHistoricOrdersData +from sysdata.csv.csv_capital_data import csvCapitalData +from sysdata.csv.csv_optimal_position import csvOptimalPositionData +from sysdata.csv.csv_instrument_config import csvFuturesInstrumentData +from sysdata.csv.csv_roll_state_storage import csvRollStateData + + def backup_arctic_to_csv(): data = dataBlob(log_name="backup_arctic_to_csv") @@ -72,23 +86,15 @@ def get_data_and_create_csv_directories(logname): csv_data_paths=class_paths, keep_original_prefix=True, log_name=logname ) - data.add_class_list( - "csvFuturesContractPriceData csvFuturesAdjustedPricesData \ - csvFuturesMultiplePricesData csvFxPricesData \ - csvContractPositionData csvStrategyPositionData \ - csvBrokerHistoricOrdersData csvContractHistoricOrdersData csvStrategyHistoricOrdersData \ - csvCapitalData csvOptimalPositionData csvFuturesInstrumentData \ - csvRollStateData csvFuturesContractData" + data.add_class_list([ + csvFuturesContractPriceData, csvFuturesAdjustedPricesData, + csvFuturesMultiplePricesData, csvFxPricesData, + csvContractPositionData, csvStrategyPositionData, + csvBrokerHistoricOrdersData, csvContractHistoricOrdersData, csvStrategyHistoricOrdersData, + csvCapitalData, csvOptimalPositionData, csvFuturesInstrumentData, + csvRollStateData, csvFuturesContractData] ) - data.add_class_list( - "arcticFuturesContractPriceData arcticFuturesMultiplePricesData \ - arcticFuturesAdjustedPricesData arcticFxPricesData \ - mongoContractPositionData mongoStrategyPositionData \ - mongoBrokerHistoricOrdersData mongoContractHistoricOrdersData mongoStrategyHistoricOrdersData \ - mongoCapitalData mongoFuturesContractData mongoFuturesInstrumentData \ - mongoOptimalPositionData mongoRollParametersData mongoRollStateData" - ) return data diff --git a/sysproduction/data/broker.py b/sysproduction/data/broker.py index 54e3853152..70a0c46a83 100644 --- a/sysproduction/data/broker.py +++ b/sysproduction/data/broker.py @@ -4,7 +4,15 @@ from collections import namedtuple +"ibFxPricesData, ibFuturesContractPriceData, ibFuturesContractData,\ +ibContractPositionData, ibOrdersData, ibMiscData]" + from sysbrokers.IB.ibSpotFXData import ibFxPricesData +from sysbrokers.IB.ibFuturesContractPriceData import ibFuturesContractPriceData +from sysbrokers.IB.ibFuturesContracts import ibFuturesContractData +from sysbrokers.IB.ibPositionData import ibContractPositionData +from sysbrokers.IB.ibOrders import ibOrdersData +from sysbrokers.IB.ibMiscData import ibMiscData from syscore.objects import missing_data, arg_not_supplied, missing_order, missing_contract @@ -33,9 +41,9 @@ def __init__(self, data=arg_not_supplied): data.add_ib_class("ibFxPricesData", ibFxPricesData) - data.add_class_list( - "ibFxPricesData ibFuturesContractPriceData ibFuturesContractData\ - ibContractPositionData ibOrdersData ibMiscData" + data.add_class_list([ + ibFxPricesData, ibFuturesContractPriceData, ibFuturesContractData, + ibContractPositionData, ibOrdersData, ibMiscData] ) self.data = data diff --git a/sysproduction/data/capital.py b/sysproduction/data/capital.py index 11c6f9bec9..eb5de28df3 100644 --- a/sysproduction/data/capital.py +++ b/sysproduction/data/capital.py @@ -1,6 +1,7 @@ from syscore.objects import missing_data, arg_not_supplied from sysdata.production.capital import totalCapitalCalculationData +from sysdata.mongodb.mongo_capital import mongoCapitalData from sysdata.private_config import get_private_then_default_key_value from sysproduction.data.get_data import dataBlob @@ -13,7 +14,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoCapitalData") + data.add_class_object(mongoCapitalData) self.data = data @property diff --git a/sysproduction/data/contracts.py b/sysproduction/data/contracts.py index d4ad793b4e..4d3a835bdf 100644 --- a/sysproduction/data/contracts.py +++ b/sysproduction/data/contracts.py @@ -2,13 +2,16 @@ from syscore.objects import missing_contract, arg_not_supplied +from sysdata.arctic.arctic_futures_per_contract_prices import arcticFuturesContractPriceData +from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData +from sysdata.mongodb.mongo_roll_data import mongoRollParametersData +from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData from sysdata.futures.rolls import contractDateWithRollParameters from sysdata.futures.contracts import futuresContract from sysproduction.data.prices import get_valid_instrument_code_from_user, diagPrices from sysproduction.data.get_data import dataBlob - missing_expiry = datetime.datetime(1900, 1, 1) @@ -18,9 +21,9 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list( - "arcticFuturesContractPriceData mongoRollParametersData \ - arcticFuturesMultiplePricesData mongoFuturesContractData" + data.add_class_list([ + arcticFuturesContractPriceData, mongoRollParametersData, + arcticFuturesMultiplePricesData, mongoFuturesContractData] ) self.data = data @@ -280,7 +283,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoFuturesContractData") + data.add_class_object(mongoFuturesContractData) self.data = data def add_contract_data(self, contract, ignore_duplication=False): diff --git a/sysproduction/data/controls.py b/sysproduction/data/controls.py index db7a665028..eb545f9d9d 100644 --- a/sysproduction/data/controls.py +++ b/sysproduction/data/controls.py @@ -8,6 +8,11 @@ from sysdata.private_config import get_private_then_default_key_value from sysdata.production.position_limits import positionLimitAndPosition, positionLimitForInstrument, positionLimitForStrategyInstrument +from sysdata.mongodb.mongo_process_control import mongoControlProcessData +from sysdata.mongodb.mongo_lock_data import mongoLockData +from sysdata.mongodb.mongo_position_limits import mongoPositionLimitData +from sysdata.mongodb.mongo_trade_limits import mongoTradeLimitData +from sysdata.mongodb.mongo_override import mongoOverrideData from sysproduction.data.get_data import dataBlob from sysproduction.data.strategies import diagStrategiesConfig @@ -20,7 +25,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoLockData") + data.add_class_object(mongoLockData) self.data = data def is_instrument_locked(self, instrument_code): @@ -42,7 +47,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoTradeLimitData") + data.add_class_object(mongoTradeLimitData) self.data = data def what_trade_is_possible( @@ -97,7 +102,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoOverrideData") + data.add_class_object(mongoOverrideData) self.data = data def get_dict_of_all_overrides(self): @@ -118,7 +123,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoOverrideData") + data.add_class_object(mongoOverrideData) self.data = data def update_override_for_strategy(self, strategy_name, new_override): @@ -152,7 +157,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoControlProcessData") + data.add_class_object(mongoControlProcessData) self.data = data def get_dict_of_control_processes(self): @@ -487,10 +492,9 @@ def __init__(self, data=arg_not_supplied): # Check data has the right elements to do this if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoPositionLimitData") + data.add_class_object(mongoPositionLimitData) self.data = data - def cut_down_proposed_instrument_trade_okay( self, instrument_trade): diff --git a/sysproduction/data/currency_data.py b/sysproduction/data/currency_data.py index bf409eb883..7766fa6322 100644 --- a/sysproduction/data/currency_data.py +++ b/sysproduction/data/currency_data.py @@ -1,8 +1,11 @@ -from sysdata.private_config import get_private_then_default_key_value -from sysproduction.data.get_data import dataBlob from syscore.objects import arg_not_supplied from syscore.genutils import print_menu_of_values_and_get_response + +from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData from sysdata.fx.spotfx import currencyValue +from sysdata.private_config import get_private_then_default_key_value + +from sysproduction.data.get_data import dataBlob class currencyData(object): @@ -15,7 +18,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("arcticFxPricesData") + data.add_class_object(arcticFxPricesData) self.data = data def update_fx_prices(self, fx_code, new_fx_prices, check_for_spike=True): diff --git a/sysproduction/data/get_data.py b/sysproduction/data/get_data.py index 989a2a02dd..f880839ee9 100644 --- a/sysproduction/data/get_data.py +++ b/sysproduction/data/get_data.py @@ -3,77 +3,18 @@ from copy import copy -from sysbrokers.IB.ibFuturesContractPriceData import ibFuturesContractPriceData -from sysbrokers.IB.ibSpotFXData import ibFxPricesData from sysbrokers.IB.ibConnection import connectionIB -from sysbrokers.IB.ibFuturesContracts import ibFuturesContractData -from sysbrokers.IB.ibPositionData import ibContractPositionData -from sysbrokers.IB.ibOrders import ibOrdersData -from sysbrokers.IB.ibMiscData import ibMiscData - -from sysdata.arctic.arctic_futures_per_contract_prices import ( - arcticFuturesContractPriceData, -) -from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData -from sysdata.arctic.arctic_adjusted_prices import arcticFuturesAdjustedPricesData -from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData -from sysdata.arctic.arctic_and_mongo_sim_futures_data import arcticFuturesSimData - -from sysdata.csv.csv_futures_contract_prices import csvFuturesContractPriceData -from sysdata.csv.csv_adjusted_prices import csvFuturesAdjustedPricesData -from sysdata.csv.csv_multiple_prices import csvFuturesMultiplePricesData -from sysdata.csv.csv_spot_fx import csvFxPricesData -from sysdata.csv.csv_contract_position_data import csvContractPositionData -from sysdata.csv.csv_strategy_position_data import csvStrategyPositionData -from sysdata.csv.csv_historic_orders import ( - csvBrokerHistoricOrdersData, - csvContractHistoricOrdersData, - csvStrategyHistoricOrdersData, -) -from sysdata.csv.csv_capital_data import csvCapitalData -from sysdata.csv.csv_optimal_position import csvOptimalPositionData -from sysdata.csv.csv_instrument_config import csvFuturesInstrumentData -from sysdata.csv.csv_roll_state_storage import csvRollStateData -from sysdata.csv.csv_futures_contracts import csvFuturesContractData - -from sysdata.mongodb.mongo_futures_instruments import mongoFuturesInstrumentData -from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData -from sysdata.mongodb.mongo_roll_data import mongoRollParametersData -from sysdata.mongodb.mongo_roll_state_storage import mongoRollStateData -from sysdata.mongodb.mongo_position_by_contract import mongoContractPositionData -from sysdata.mongodb.mongo_capital import mongoCapitalData -from sysdata.mongodb.mongo_optimal_position import mongoOptimalPositionData -from sysdata.mongodb.mongo_positions_by_strategy import mongoStrategyPositionData -from sysdata.mongodb.mongo_position_limits import mongoPositionLimitData -from sysdata.mongodb.mongo_order_stack import ( - mongoInstrumentOrderStackData, - mongoContractOrderStackData, - mongoBrokerOrderStackData, -) -from sysdata.mongodb.mongo_historic_orders import ( - mongoStrategyHistoricOrdersData, - mongoContractHistoricOrdersData, - mongoBrokerHistoricOrdersData, -) -from sysdata.mongodb.mongo_override import mongoOverrideData -from sysdata.mongodb.mongo_trade_limits import mongoTradeLimitData -from sysdata.mongodb.mongo_lock_data import mongoLockData -from sysdata.mongodb.mongo_process_control import mongoControlProcessData -from sysdata.mongodb.mongo_log import mongoLogData -from sysdata.mongodb.mongo_email_control import mongoEmailControlData - -from sysdata.mongodb.mongo_connection import mongoDb from sysdata.mongodb.mongo_connection import mongoDb from sysdata.mongodb.mongo_log import logToMongod as logger -from syscore.objects import arg_not_supplied, success, failure +from syscore.objects import arg_not_supplied class dataBlob(object): def __init__( self, - arg_string=arg_not_supplied, + class_list=arg_not_supplied, log_name="", csv_data_paths=arg_not_supplied, ib_conn=arg_not_supplied, @@ -87,7 +28,7 @@ def __init__( Class names we know how to handle are: 'ib*', 'mongo*', 'arctic*', 'csv*' - data = dataBlob("arcticFuturesContractPriceData arcticFuturesContractPriceData mongoFuturesContractData') + data = dataBlob([arcticFuturesContractPriceData, arcticFuturesContractPriceData, mongoFuturesContractData]) .... sets up the following equivalencies: @@ -103,7 +44,7 @@ def __init__( :param log_name: logger type to set :param keep_original_prefix: bool. If True then: - data = dataBlob("arcticFuturesContractPriceData arcticFuturesContractPriceData mongoFuturesContractData') + data = dataBlob([arcticFuturesContractPriceData, arcticFuturesContractPriceData, mongoFuturesContractData]) .... sets up the following equivalencies. This is useful if you are copying from one source to another @@ -122,150 +63,116 @@ def __init__( self._log = log self._log_name = log_name self._csv_data_paths = csv_data_paths - self.keep_original_prefix = keep_original_prefix - self.attr_list = [] - self.class_list = [] + self._keep_original_prefix = keep_original_prefix - if arg_string is arg_not_supplied: + self._attr_list = [] + + if class_list is arg_not_supplied: # can set up dynamically later pass else: - self.add_class_list(arg_string) + self.add_class_list(class_list) self._original_data = copy(self) def __repr__(self): - return "dataBlob with elements: %s" % ",".join(self.attr_list) + return "dataBlob with elements: %s" % ",".join(self._attr_list) - """ - Following two methods implement context manager - """ - def __enter__(self): - return self + def add_class_list(self, class_list: list): + for class_object in class_list: + self.add_class_object(class_object) - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() + def add_class_object(self, class_object): + resolved_instance = self._get_resolved_instance_of_class(class_object) + class_name = get_class_name(class_object) + self._resolve_names_and_add(resolved_instance, class_name) - def close(self): - if self._ib_conn is not arg_not_supplied: - self.ib_conn.close_connection() + def _get_resolved_instance_of_class(self, class_object): + class_adding_method = self._get_class_adding_method(class_object) + resolved_instance = class_adding_method(class_object) - if self._mongo_db is not arg_not_supplied: - self.mongo_db.close() + return resolved_instance - @property - def ib_conn(self): - ib_conn = getattr(self, "_ib_conn", arg_not_supplied) - if ib_conn is arg_not_supplied: - ib_conn = connectionIB() - self._ib_conn = ib_conn - return ib_conn + def _get_class_adding_method(self, class_object): + prefix = self._get_class_prefix(class_object) + class_dict = dict(ib = self._add_ib_class, csv = self._add_csv_class, arctic = self._add_arctic_class, + mongo = self._add_mongo_class) - @property - def mongo_db(self): - mongo_db = getattr(self, "_mongo_db", arg_not_supplied) - if mongo_db is arg_not_supplied: - mongo_db = mongoDb() - self._mongo_db = mongo_db + method_to_add_with = class_dict.get(prefix, None) + if method_to_add_with is None: + error_msg = "Don't know how to handle object named %s" % get_class_name(class_object) + self._raise_and_log_error(error_msg) - return mongo_db + return method_to_add_with - @property - def log(self): - log = getattr(self, "_log", arg_not_supplied) - if log is arg_not_supplied: - log = logger(self._log_name, data=self, mongo_db=self.mongo_db) - log.set_logging_level("on") - self._log = log - - return log - - @property - def log_name(self): - return self.log.attributes["type"] + def _get_class_prefix(self, class_object) -> str: + class_name = get_class_name(class_object) + split_up_name = camel_case_split(class_name) + prefix = split_up_name[0] + return prefix - @property - def csv_data_paths(self): - csv_data_paths = getattr(self, "_csv_data_paths", arg_not_supplied) - if csv_data_paths is arg_not_supplied: - raise Exception("No defaults for csv data paths") - return csv_data_paths - - ## NEW STYLE CLASS STUFF - def add_ib_class(self, class_name: str, class_object): + def _add_ib_class(self, class_object): + log = self._get_specific_logger(class_object) try: - resolved_instance = class_object(self.ib_conn, log = self.log.setup(component = class_name)) + resolved_instance = class_object(self.ib_conn, log = log) except Exception as e: + class_name = get_class_name(class_object) msg = ( - "Couldn't evaluate %s(self.ib_conn, log = self.log.setup(component = %s)) This might be because (a) IB gateway not running, or (b) import is missing\ - or (c) arguments don't follow pattern" % (class_name, class_name)) - self.log.critical(msg) - raise Exception(msg) + "Error %s couldn't evaluate %s(self.ib_conn, log = self.log.setup(component = %s)) This might be because (a) IB gateway not running, or (b) import is missing\ + or (c) arguments don't follow pattern" % (str(e), class_name, class_name)) + self._raise_and_log_error(msg) - self.resolve_names_and_add(resolved_instance, class_name) + return resolved_instance - def add_mongo_class(self, class_name: str, class_object): + def _add_mongo_class(self, class_object): + log = self._get_specific_logger(class_object) try: - resolved_instance = class_object(mongo_db=self.mongo_db, log = self.log.setup(component = class_name)) + resolved_instance = class_object(mongo_db=self.mongo_db, log = log) except Exception as e: + class_name = get_class_name(class_object) msg = ( - "Couldn't evaluate %s(mongo_db=self.mongo_db, log = self.log.setup(component = %s)) \ + "Error %s couldn't evaluate %s(mongo_db=self.mongo_db, log = self.log.setup(component = %s)) \ This might be because import is missing\ - or arguments don't follow pattern" % (class_name, class_name)) - self.log.critical(msg) - raise Exception(msg) + or arguments don't follow pattern" % (str(e), class_name, class_name)) + self._raise_and_log_error(msg) - self.resolve_names_and_add(resolved_instance, class_name) + return resolved_instance - def add_arctic_class(self, class_name: str, class_object): + def _add_arctic_class(self, class_object): + log = self._get_specific_logger(class_object) try: - resolved_instance = class_object(mongo_db=self.mongo_db, log = self.log.setup(component = class_name)) + resolved_instance = class_object(mongo_db=self.mongo_db, log = log) except Exception as e: + class_name = get_class_name(class_object) msg = ( - "Couldn't evaluate %s(mongo_db=self.mongo_db, log = self.log.setup(component = %s)) \ + "Error %s couldn't evaluate %s(mongo_db=self.mongo_db, log = self.log.setup(component = %s)) \ This might be because import is missing\ - or arguments don't follow pattern" % (class_name, class_name)) - self.log.critical(msg) - raise Exception(msg) + or arguments don't follow pattern" % (str(e), class_name, class_name)) + self._raise_and_log_error(msg) - self.resolve_names_and_add(resolved_instance, class_name) + return resolved_instance + + def _add_csv_class(self, class_object): + datapath = self._get_csv_paths_for_class(class_object) + log = self._get_specific_logger(class_object) - def add_csv_class(self, class_name: str, class_object): - datapath = self.get_csv_paths_for_class(class_name) try: - resolved_instance = class_object(datapath = datapath, log = self.log.setup(component = class_name)) + resolved_instance = class_object(datapath = datapath, log = log) except Exception as e: + class_name = get_class_name(class_object) msg = ( - "Couldn't evaluate %s(datapath = datapath, log = self.log.setup(component = %s)) \ + "Error %s couldn't evaluate %s(datapath = datapath, log = self.log.setup(component = %s)) \ This might be because import is missing\ - or arguments don't follow pattern" % (class_name, class_name)) - self.log.critical(msg) - raise Exception(msg) - - self.resolve_names_and_add(resolved_instance, class_name) - - - def resolve_names_and_add(self, resolved_instance, class_name: str): - attr_name = self.get_new_name(class_name) - self.add_new_class_ids(resolved_instance, attr_name, class_name) - - def get_new_name(self, class_name: str): - split_up_name = camel_case_split(class_name) - attr_name = identifying_name( - split_up_name, keep_original_prefix=self.keep_original_prefix - ) + or arguments don't follow pattern" % (str(e), class_name, class_name)) + self._raise_and_log_error(msg) - return attr_name + return resolved_instance - def add_new_class_ids(self, resolved_instance, attr_name:str, class_name: str): - setattr(self, attr_name, resolved_instance) - self.attr_list.append(attr_name) - self.class_list.append(class_name) - - def get_csv_paths_for_class(self, class_name): + def _get_csv_paths_for_class(self, class_object): + class_name = get_class_name(class_object) csv_data_paths = self.csv_data_paths if csv_data_paths is arg_not_supplied: raise Exception( @@ -279,76 +186,93 @@ def get_csv_paths_for_class(self, class_name): return datapath - ## OLD STYLE CLASS NAMES DEPRECATED - def add_class_list(self, arg_string): - list_of_classes = arg_string.split(" ") + @property + def csv_data_paths(self) -> dict: + csv_data_paths = getattr(self, "_csv_data_paths", arg_not_supplied) + if csv_data_paths is arg_not_supplied: + raise Exception("No defaults for csv data paths") + return csv_data_paths + + def _get_specific_logger(self, class_object): + class_name = get_class_name(class_object) + log = self.log.setup(component = class_name) - for class_name in list_of_classes: - self._add_class_element(class_name) - return success + return log - ## OLD STYLE CLASS NAMES DEPRECATED - def _add_class_element(self, class_name): - if class_name in self.class_list: - # Already present - return success + def _resolve_names_and_add(self, resolved_instance, class_name: str): + attr_name = self._get_new_name(class_name) + self._add_new_class_with_new_name(resolved_instance, attr_name) - if len(class_name) == 0: - return failure + def _get_new_name(self, class_name: str): + split_up_name = camel_case_split(class_name) + attr_name = identifying_name( + split_up_name, keep_original_prefix=self._keep_original_prefix + ) + + return attr_name - attr_name, resolved_instance = self.process_class_id(class_name) + def _add_new_class_with_new_name(self, resolved_instance, attr_name:str): setattr(self, attr_name, resolved_instance) + self._add_attr_to_list(attr_name) - self.attr_list.append(attr_name) - self.class_list.append(class_name) - return success + def _add_attr_to_list(self, new_attr: str): + self._attr_list.append(new_attr) - ## OLD STYLE CLASS NAMES DEPRECATED - def process_class_id(self, class_name): - """ + """ + Following two methods implement context manager + """ - :param class_name: name of class to add to data + def __enter__(self): + return self - :return: 2 tuple: identifying attribute name str, instance of class - """ + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() - split_up_name = camel_case_split(class_name) - prefix = split_up_name[0] + def close(self): + if self._ib_conn is not arg_not_supplied: + self.ib_conn.close_connection() - # NEED TO DYNAMICALLY SWITCH DB SOURCE AND ADD DEFAULT CSV PATH NAME - # REMOVE SOURCE_ PREFIX, REPLACE WITH db_ or broker_ - if prefix == "csv": - datapath = self.get_csv_paths_for_class(class_name) + if self._mongo_db is not arg_not_supplied: + self.mongo_db.close() - eval_dict = dict( - ib="%s(self.ib_conn, log=self.log.setup(component='%s'))", - mongo="%s(mongo_db=self.mongo_db, log=self.log.setup(component='%s'))", - arctic="%s(mongo_db=self.mongo_db, log=self.log.setup(component='%s'))", - csv="%s(datapath=datapath, log=self.log.setup(component='%s'))", - ) + @property + def ib_conn(self): + ib_conn = getattr(self, "_ib_conn", arg_not_supplied) + if ib_conn is arg_not_supplied: + ib_conn = connectionIB() + self._ib_conn = ib_conn - to_eval = eval_dict[prefix] % ( - class_name, - class_name, - ) # class_name appears twice as always passed as a log - # The eval may use ib_conn, mongo_db, datapath and will always use log - try: - resolved_instance = eval(to_eval) - except BaseException: - msg = ( - "Couldn't evaluate %s This might be because (a) IB gateway not running, or (b) it is missing from sysproduction.data.get_data imports or arguments don't follow pattern" % - to_eval) - self.log.critical(msg) - raise Exception(msg) + return ib_conn - keep_original_prefix = self.keep_original_prefix + @property + def mongo_db(self): + mongo_db = getattr(self, "_mongo_db", arg_not_supplied) + if mongo_db is arg_not_supplied: + mongo_db = mongoDb() + self._mongo_db = mongo_db - attr_name = identifying_name( - split_up_name, keep_original_prefix=keep_original_prefix - ) + return mongo_db + + def _raise_and_log_error(self, error_msg): + self.log.critical(error_msg) + raise Exception(error_msg) + + @property + def log(self): + log = getattr(self, "_log", arg_not_supplied) + if log is arg_not_supplied: + log = logger(self.log_name, data=self, mongo_db=self.mongo_db) + log.set_logging_level("on") + self._log = log + + return log + + @property + def log_name(self) -> str: + log_name = getattr(self, "_logname", "") + return log_name - return attr_name, resolved_instance source_dict = dict(arctic="db", mongo="db", csv="db", ib="broker") @@ -399,3 +323,6 @@ def camel_case_split(str): words[-1].append(c) return ["".join(word) for word in words] + +def get_class_name(class_object): + return class_object.__name__ \ No newline at end of file diff --git a/sysproduction/data/instruments.py b/sysproduction/data/instruments.py index 7a499e18cf..2cf486312e 100644 --- a/sysproduction/data/instruments.py +++ b/sysproduction/data/instruments.py @@ -1,7 +1,10 @@ import datetime -from sysproduction.data.get_data import dataBlob from syscore.objects import arg_not_supplied + +from sysdata.mongodb.mongo_futures_instruments import mongoFuturesInstrumentData + +from sysproduction.data.get_data import dataBlob from sysproduction.data.currency_data import currencyData, currencyValue @@ -11,7 +14,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list(" mongoFuturesInstrumentData") + data.add_class_object(mongoFuturesInstrumentData) self.data = data def get_point_size(self, instrument_code): diff --git a/sysproduction/data/logs.py b/sysproduction/data/logs.py index cac4abe129..1d178b655b 100644 --- a/sysproduction/data/logs.py +++ b/sysproduction/data/logs.py @@ -1,5 +1,6 @@ -from sysproduction.data.get_data import dataBlob from syscore.objects import arg_not_supplied +from sysdata.mongodb.mongo_log import mongoLogData +from sysproduction.data.get_data import dataBlob class diagLogs(object): @@ -8,7 +9,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoLogData") + data.add_class_object(mongoLogData) self.data = data def get_log_items(self, attribute_dict=dict(), lookback_days=1): diff --git a/sysproduction/data/orders.py b/sysproduction/data/orders.py index 0574cfd0d0..de0b0645e8 100644 --- a/sysproduction/data/orders.py +++ b/sysproduction/data/orders.py @@ -1,12 +1,13 @@ from syscore.objects import ( arg_not_supplied, missing_data, - success, - failure, no_parent, missing_order, ) +from sysdata.mongodb.mongo_order_stack import mongoInstrumentOrderStackData, mongoContractOrderStackData, mongoBrokerOrderStackData +from sysdata.mongodb.mongo_historic_orders import mongoStrategyHistoricOrdersData, mongoContractHistoricOrdersData, mongoBrokerHistoricOrdersData + from sysproduction.data.get_data import dataBlob @@ -15,11 +16,9 @@ def __init__(self, data=arg_not_supplied): # Check data has the right elements to do this if data is arg_not_supplied: data = dataBlob() - data.add_class_list( - "mongoInstrumentOrderStackData mongoContractOrderStackData mongoBrokerOrderStackData" - ) - data.add_class_list( - "mongoContractHistoricOrdersData mongoStrategyHistoricOrdersData mongoBrokerHistoricOrdersData" + data.add_class_list([mongoInstrumentOrderStackData, mongoContractOrderStackData, + mongoBrokerOrderStackData, mongoContractHistoricOrdersData, + mongoStrategyHistoricOrdersData, mongoBrokerHistoricOrdersData] ) self.data = data diff --git a/sysproduction/data/positions.py b/sysproduction/data/positions.py index 864cdb3197..b6b0aae10a 100644 --- a/sysproduction/data/positions.py +++ b/sysproduction/data/positions.py @@ -8,6 +8,11 @@ missing_order, ) +from sysdata.mongodb.mongo_roll_state_storage import mongoRollStateData +from sysdata.mongodb.mongo_position_by_contract import mongoContractPositionData +from sysdata.mongodb.mongo_positions_by_strategy import mongoStrategyPositionData +from sysdata.mongodb.mongo_optimal_position import mongoOptimalPositionData + from sysproduction.data.contracts import missing_contract from sysproduction.data.get_data import dataBlob @@ -18,8 +23,8 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list( - "mongoRollStateData mongoContractPositionData mongoStrategyPositionData" + data.add_class_list([ + mongoRollStateData, mongoContractPositionData, mongoStrategyPositionData] ) self.data = data self.log = data.log @@ -150,7 +155,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("mongoOptimalPositionData") + data.add_class_object(mongoOptimalPositionData) self.data = data self.log = data.log @@ -209,7 +214,7 @@ def __init__(self, data=arg_not_supplied): data = dataBlob() data.add_class_list( - "mongoRollStateData mongoContractPositionData mongoStrategyPositionData mongoOptimalPositionData" + [mongoRollStateData, mongoContractPositionData, mongoStrategyPositionData, mongoOptimalPositionData] ) self.data = data self.log = data.log diff --git a/sysproduction/data/prices.py b/sysproduction/data/prices.py index 17cbac9e3a..8953d69725 100644 --- a/sysproduction/data/prices.py +++ b/sysproduction/data/prices.py @@ -1,10 +1,18 @@ +import numpy as np + +from syscore.objects import missing_contract, arg_not_supplied, missing_data + from sysdata.futures.futures_per_contract_prices import dictFuturesContractPrices from sysdata.private_config import get_private_then_default_key_value -from syscore.dateutils import ROOT_BDAYS_INYEAR +from sysdata.arctic.arctic_futures_per_contract_prices import arcticFuturesContractPriceData +from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData +from sysdata.arctic.arctic_adjusted_prices import arcticFuturesAdjustedPricesData +from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData + from sysproduction.data.get_data import dataBlob -from syscore.objects import missing_contract, arg_not_supplied, missing_data -import numpy as np + + class diagPrices(object): @@ -13,9 +21,9 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list( - "arcticFuturesContractPriceData arcticFuturesAdjustedPricesData \ - arcticFuturesMultiplePricesData mongoFuturesContractData " + data.add_class_list([ + arcticFuturesContractPriceData, arcticFuturesAdjustedPricesData, + arcticFuturesMultiplePricesData, mongoFuturesContractData] ) self.data = data @@ -120,9 +128,9 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list( - "arcticFuturesContractPriceData arcticFuturesMultiplePricesData \ - mongoFuturesContractData arcticFuturesAdjustedPricesData" + data.add_class_list([ + arcticFuturesContractPriceData, arcticFuturesMultiplePricesData, + mongoFuturesContractData, arcticFuturesAdjustedPricesData] ) self.data = data diff --git a/sysproduction/data/sim_data.py b/sysproduction/data/sim_data.py index b01db6bb0b..db88eccbd5 100644 --- a/sysproduction/data/sim_data.py +++ b/sysproduction/data/sim_data.py @@ -1,7 +1,7 @@ from syscore.objects import arg_not_supplied -from syscore.genutils import get_and_convert + +from sysdata.arctic.arctic_and_mongo_sim_futures_data import arcticFuturesSimData from sysproduction.data.get_data import dataBlob -from sysdata.private_config import get_private_then_default_key_value class dataSimData(object): @@ -10,7 +10,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("arcticFuturesSimData") + data.add_class_object(arcticFuturesSimData) self.data = data def sim_data(self): diff --git a/sysproduction/data/volumes.py b/sysproduction/data/volumes.py index 4b31bbc83d..0063be1cd1 100644 --- a/sysproduction/data/volumes.py +++ b/sysproduction/data/volumes.py @@ -1,7 +1,9 @@ import datetime as datetime -from sysproduction.data.get_data import dataBlob from syscore.objects import missing_contract, arg_not_supplied, missing_data +from sysdata.arctic.arctic_futures_per_contract_prices import arcticFuturesContractPriceData +from sysproduction.data.get_data import dataBlob + # Get volume data for the contract we're currently trading, plus what we might roll into, plus the previous one # This is handy for working out whether to roll @@ -13,7 +15,7 @@ def __init__(self, data=arg_not_supplied): if data is arg_not_supplied: data = dataBlob() - data.add_class_list("arcticFuturesContractPriceData") + data.add_class_object(arcticFuturesContractPriceData) self.data = data def get_normalised_smoothed_volumes_of_contract_list( @@ -22,7 +24,6 @@ def get_normalised_smoothed_volumes_of_contract_list( """ :param instrument_code: - :param data: :return: dict, keys are contract names Values are normalised volumes, with largest volume contract as 1.0 """ diff --git a/sysproduction/diagnostic/emailing.py b/sysproduction/diagnostic/emailing.py index 2132ca3d1b..4544a272b8 100644 --- a/sysproduction/diagnostic/emailing.py +++ b/sysproduction/diagnostic/emailing.py @@ -1,8 +1,8 @@ import datetime from syscore.dateutils import SECONDS_PER_DAY -from syscore.objects import arg_not_supplied from syslogdiag.emailing import send_mail_msg +from sysdata.mongodb.mongo_email_control import mongoEmailControlData def send_production_mail_msg(data, body, subject, report=False): """ @@ -134,7 +134,7 @@ def __init__(self, data): # Check data has the right elements to do this # uniquely, we don't allow a default data or this causes circular # imports - data.add_class_list("mongoEmailControlData") + data.add_class_list([mongoEmailControlData]) self.data = data def get_time_last_email_sent_with_this_subject(self, subject):