From 6856479949fcd55dd29b5f31fb46d35f50d8936a Mon Sep 17 00:00:00 2001 From: Rob Carver Date: Mon, 20 Nov 2023 10:34:13 +0000 Subject: [PATCH] much cleaner way to switch arctic/parquet --- sysdata/data_blob.py | 19 ++- sysdata/pointers.py | 4 - sysdata/sim/db_futures_sim_data.py | 39 ++++- ...stedprices_from_mongo_multiple_to_mongo.py | 4 +- sysinit/futures/clone_data_for_instrument.py | 2 + sysproduction/backup_arctic_to_csv.py | 157 ++++++++---------- sysproduction/data/capital.py | 5 +- sysproduction/data/contracts.py | 16 +- sysproduction/data/control_process.py | 5 +- sysproduction/data/currency_data.py | 5 +- sysproduction/data/instruments.py | 12 +- sysproduction/data/optimal_positions.py | 5 +- sysproduction/data/orders.py | 24 +-- sysproduction/data/positions.py | 14 +- sysproduction/data/prices.py | 24 +-- sysproduction/data/production_data_objects.py | 88 ++++++++++ sysproduction/data/sim_data.py | 19 +-- sysproduction/data/volumes.py | 4 +- 18 files changed, 259 insertions(+), 187 deletions(-) create mode 100644 sysproduction/data/production_data_objects.py diff --git a/sysdata/data_blob.py b/sysdata/data_blob.py index 485f915e75..ba095d0b3d 100644 --- a/sysdata/data_blob.py +++ b/sysdata/data_blob.py @@ -80,13 +80,13 @@ def __init__( def __repr__(self): return "dataBlob with elements: %s" % ",".join(self._attr_list) - def add_class_list(self, class_list: list): + def add_class_list(self, class_list: list, use_prefix: str = arg_not_supplied): for class_object in class_list: - self.add_class_object(class_object) + self.add_class_object(class_object, use_prefix=use_prefix) - def add_class_object(self, class_object): + def add_class_object(self, class_object, use_prefix: str = arg_not_supplied): class_name = get_class_name(class_object) - attr_name = self._get_new_name(class_name) + attr_name = self._get_new_name(class_name, use_prefix=use_prefix) if not self._already_existing_class_name(attr_name): resolved_instance = self._get_resolved_instance_of_class(class_object) self._resolve_names_and_add(resolved_instance, class_name) @@ -236,10 +236,11 @@ def _resolve_names_and_add(self, resolved_instance, class_name: str): attr_name = self._get_new_name(class_name) self._add_new_class_with_new_name(resolved_instance, attr_name) - def _get_new_name(self, class_name: str) -> str: + def _get_new_name(self, class_name: str, use_prefix: str = arg_not_supplied) -> str: split_up_name = camel_case_split(class_name) attr_name = identifying_name( - split_up_name, keep_original_prefix=self._keep_original_prefix + split_up_name, keep_original_prefix=self._keep_original_prefix, + use_prefix=use_prefix ) return attr_name @@ -382,7 +383,7 @@ def get_parquet_root_directory(config): return get_resolved_pathname(path) -def identifying_name(split_up_name: list, keep_original_prefix=False) -> str: +def identifying_name(split_up_name: list, keep_original_prefix: bool=False, use_prefix: str = arg_not_supplied) -> str: """ Turns sourceClassNameData into broker_class_name or db_class_name @@ -400,7 +401,9 @@ def identifying_name(split_up_name: list, keep_original_prefix=False) -> str: except BaseException: raise Exception("Get_data strings only work if class name ends in ...Data") - if keep_original_prefix: + if use_prefix is not arg_not_supplied: + source_label = use_prefix + elif keep_original_prefix: source_label = original_source_label else: try: diff --git a/sysdata/pointers.py b/sysdata/pointers.py index 5a20d08149..512afa7de5 100644 --- a/sysdata/pointers.py +++ b/sysdata/pointers.py @@ -1,10 +1,7 @@ -from sysdata.parquet.parquet_adjusted_prices import parquetFuturesAdjustedPricesData as og_parquetFuturesAdjustedPricesData from sysdata.arctic.arctic_adjusted_prices import arcticFuturesAdjustedPricesData -from sysdata.parquet.parquet_capital import parquetCapitalData as og_parquetCapitalData from sysdata.arctic.arctic_capital import arcticCapitalData from sysdata.arctic.arctic_futures_per_contract_prices import arcticFuturesContractPriceData -from sysdata.parquet.parquet_futures_per_contract_prices import parquetFuturesContractPriceData as og_parquetFuturesContractPriceData from sysdata.data_blob import get_parquet_root_directory from sysdata.config.production_config import get_production_config @@ -15,7 +12,6 @@ pass ## TO USE ARCTIC RATHER THAN PARQUET, REPLACE THE og_ with the relevant arctic class -parquetFuturesAdjustedPricesData = og_parquetFuturesAdjustedPricesData ## change to arctic if desired parquet_futures_adjusted_price_data = parquetFuturesAdjustedPricesData(parquet_root) ## replace with arcticFuturesContractPriceData() if desired parquetCapitalData = og_parquetCapitalData diff --git a/sysdata/sim/db_futures_sim_data.py b/sysdata/sim/db_futures_sim_data.py index 891636e506..9f120061f9 100644 --- a/sysdata/sim/db_futures_sim_data.py +++ b/sysdata/sim/db_futures_sim_data.py @@ -5,7 +5,7 @@ from syscore.constants import arg_not_supplied -from sysdata.pointers import parquetFuturesAdjustedPricesData +from sysdata.parquet.parquet_adjusted_prices import parquetFuturesAdjustedPricesData from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData from sysdata.csv.csv_instrument_data import csvFuturesInstrumentData @@ -26,12 +26,12 @@ def __init__( data = dataBlob( log=log, class_list=[ - parquetFuturesAdjustedPricesData, - arcticFuturesMultiplePricesData, - arcticFxPricesData, - csvFuturesInstrumentData, - csvRollParametersData, - mongoSpreadCostData, + get_class_for_data_type(FUTURES_ADJUSTED_PRICE_DATA), + get_class_for_data_type(FUTURES_MULTIPLE_PRICE_DATA), + get_class_for_data_type(FX_DATA), + get_class_for_data_type(FUTURES_INSTRUMENT_DATA), + get_class_for_data_type(ROLL_PARAMETERS_DATA), + get_class_for_data_type(SPREAD_DATA) ], ) @@ -43,6 +43,31 @@ def __repr__(self): ) +FUTURES_MULTIPLE_PRICE_DATA = "futures_multiple_price_data" +FUTURES_ADJUSTED_PRICE_DATA = "futures_adjusted_price_data" +CAPITAL_DATA = "capital_data" +FX_DATA = "fx_data" +ROLL_PARAMETERS_DATA = "roll_parameters_data" +FUTURES_INSTRUMENT_DATA = "futures_instrument_data" +SPREAD_DATA = "spread_data" + +def get_class_for_data_type(data_type:str): + + return use_sim_classes[data_type] + +use_sim_classes = { + FX_DATA: arcticFxPricesData, + ROLL_PARAMETERS_DATA: csvRollParametersData, + FUTURES_INSTRUMENT_DATA: csvFuturesInstrumentData, + + FUTURES_MULTIPLE_PRICE_DATA: arcticFuturesMultiplePricesData, + FUTURES_ADJUSTED_PRICE_DATA: parquetFuturesAdjustedPricesData, + SPREAD_DATA: mongoSpreadCostData +} + + + + if __name__ == "__main__": import doctest diff --git a/sysinit/futures/adjustedprices_from_mongo_multiple_to_mongo.py b/sysinit/futures/adjustedprices_from_mongo_multiple_to_mongo.py index bbcb988597..96663744ef 100755 --- a/sysinit/futures/adjustedprices_from_mongo_multiple_to_mongo.py +++ b/sysinit/futures/adjustedprices_from_mongo_multiple_to_mongo.py @@ -6,7 +6,7 @@ """ from syscore.constants import arg_not_supplied from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData -from sysdata.pointers import parquet_futures_adjusted_price_data +from sysproduction.data.production_data_objects import get_class_for_data_type, FUTURES_ADJUSTED_PRICE_DATA from sysdata.csv.csv_adjusted_prices import csvFuturesAdjustedPricesData from sysobjects.adjusted_prices import futuresAdjustedPrices @@ -14,7 +14,7 @@ def _get_data_inputs(csv_adj_data_path): arctic_multiple_prices = arcticFuturesMultiplePricesData() - parquet_adjusted_prices = parquet_futures_adjusted_price_data + parquet_adjusted_prices = get_class_for_data_type(FUTURES_ADJUSTED_PRICE_DATA) csv_adjusted_prices = csvFuturesAdjustedPricesData(csv_adj_data_path) return arctic_multiple_prices, parquet_adjusted_prices, csv_adjusted_prices diff --git a/sysinit/futures/clone_data_for_instrument.py b/sysinit/futures/clone_data_for_instrument.py index 8a7689f6fe..96a1b2ab4f 100644 --- a/sysinit/futures/clone_data_for_instrument.py +++ b/sysinit/futures/clone_data_for_instrument.py @@ -5,6 +5,8 @@ from sysdata.csv.csv_multiple_prices import csvFuturesMultiplePricesData from sysdata.csv.csv_adjusted_prices import csvFuturesAdjustedPricesData + + from sysobjects.contracts import futuresContract from syscore.dateutils import DAILY_PRICE_FREQ, HOURLY_FREQ from sysobjects.multiple_prices import futuresMultiplePrices diff --git a/sysproduction/backup_arctic_to_csv.py b/sysproduction/backup_arctic_to_csv.py index 24a2da5c6e..6162505051 100644 --- a/sysproduction/backup_arctic_to_csv.py +++ b/sysproduction/backup_arctic_to_csv.py @@ -24,25 +24,6 @@ from sysdata.csv.csv_roll_state_storage import csvRollStateData from sysdata.csv.csv_spreads import csvSpreadsForInstrumentData -from sysdata.pointers import parquetFuturesAdjustedPricesData -from sysdata.pointers import parquetCapitalData -from sysdata.pointers import parquetFuturesContractPriceData - -from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData -from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData -from sysdata.arctic.arctic_spreads import arcticSpreadsForInstrumentData -from sysdata.arctic.arctic_historic_strategy_positions import arcticStrategyPositionData -from sysdata.arctic.arctic_historic_contract_positions import arcticContractPositionData -from sysdata.arctic.arctic_optimal_positions import arcticOptimalPositionData - -from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData -from sysdata.mongodb.mongo_historic_orders import ( - mongoBrokerHistoricOrdersData, - mongoContractHistoricOrdersData, - mongoStrategyHistoricOrdersData, -) -from sysdata.mongodb.mongo_spread_costs import mongoSpreadCostData -from sysdata.mongodb.mongo_roll_state_storage import mongoRollStateData from sysobjects.contracts import futuresContract from sysobjects.production.tradeable_object import instrumentStrategy @@ -50,11 +31,12 @@ from sysproduction.data.directories import get_csv_backup_directory, get_csv_dump_dir from sysproduction.data.strategies import get_list_of_strategies +from sysproduction.data.production_data_objects import * -def backup_arctic_to_csv(): - data = dataBlob(log_name="backup_arctic_to_csv") +def backup_db_to_csv(): + data = dataBlob(log_name="backup_db_to_csv") backup_object = backupArcticToCsv(data) - backup_object.backup_arctic_to_csv() + backup_object.backup_db_to_csv() return None @@ -70,7 +52,7 @@ class backupArcticToCsv: def __init__(self, data): self.data = data - def backup_arctic_to_csv(self): + def backup_db_to_csv(self): backup_data = get_data_and_create_csv_directories(self.data.log_name) log = self.data.log @@ -121,7 +103,7 @@ def get_data_and_create_csv_directories(logname): os.makedirs(dir_name) data = dataBlob( - csv_data_paths=class_paths, keep_original_prefix=True, log_name=logname + csv_data_paths=class_paths, log_name=logname ) data.add_class_list( @@ -142,26 +124,29 @@ def get_data_and_create_csv_directories(logname): csvStrategyHistoricOrdersData, csvStrategyPositionData, ] + , use_prefix="csv" ) data.add_class_list( [ - parquetCapitalData, - parquetFuturesAdjustedPricesData, - parquetFuturesContractPriceData, - arcticFuturesMultiplePricesData, - arcticFxPricesData, - arcticSpreadsForInstrumentData, - mongoBrokerHistoricOrdersData, - mongoContractHistoricOrdersData, - arcticContractPositionData, - mongoFuturesContractData, - arcticOptimalPositionData, - mongoRollStateData, - mongoSpreadCostData, - mongoStrategyHistoricOrdersData, - arcticStrategyPositionData, - ] + get_class_for_data_type(CAPITAL_DATA), + get_class_for_data_type(FUTURES_ADJUSTED_PRICE_DATA), + get_class_for_data_type(FUTURES_CONTRACT_PRICE_DATA), + get_class_for_data_type(FUTURES_MULTIPLE_PRICE_DATA), + get_class_for_data_type(FX_DATA), + get_class_for_data_type(SPREAD_DATA), + get_class_for_data_type(BROKER_HISTORIC_ORDERS_DATA), + get_class_for_data_type(CONTRACT_HISTORIC_ORDERS_DATA), + get_class_for_data_type(STRATEGY_HISTORIC_ORDERS_DATA), + get_class_for_data_type(CONTRACT_POSITION_DATA), + get_class_for_data_type(STRATEGY_POSITION_DATA), + get_class_for_data_type(FUTURES_CONTRACT_DATA), + get_class_for_data_type(OPTIMAL_POSITION_DATA), + get_class_for_data_type(ROLL_STATE_DATA), + get_class_for_data_type(SPREAD_DATA) + + ], + use_prefix="db" ) return data @@ -174,7 +159,7 @@ def get_data_and_create_csv_directories(logname): # Futures contract data def backup_futures_contract_prices_to_csv(data, ignore_long_expired: bool = True): instrument_list = ( - data.arctic_futures_contract_price.get_list_of_instrument_codes_with_merged_price_data() + data.db_futures_contract_price.get_list_of_instrument_codes_with_merged_price_data() ) for instrument_code in instrument_list: backup_futures_contract_prices_for_instrument_to_csv( @@ -187,7 +172,7 @@ def backup_futures_contract_prices_to_csv(data, ignore_long_expired: bool = True def backup_futures_contract_prices_for_instrument_to_csv( data: dataBlob, instrument_code: str, ignore_long_expired: bool = True ): - list_of_contracts = data.parquet_futures_contract_price.contracts_with_merged_price_data_for_instrument_code( + list_of_contracts = data.db_futures_contract_price.contracts_with_merged_price_data_for_instrument_code( instrument_code ) @@ -209,8 +194,8 @@ def backup_futures_contract_prices_for_contract_to_csv( return None - parquet_data = ( - data.parquet_futures_contract_price.get_merged_prices_for_contract_object( + db_data = ( + data.db_futures_contract_price.get_merged_prices_for_contract_object( futures_contract ) ) @@ -219,7 +204,7 @@ def backup_futures_contract_prices_for_contract_to_csv( futures_contract ) - if check_df_equals(parquet_data, csv_data): + if check_df_equals(db_data, csv_data): # No update needed, move on data.log.debug("No prices backup needed for %s" % str(futures_contract)) else: @@ -227,7 +212,7 @@ def backup_futures_contract_prices_for_contract_to_csv( try: data.csv_futures_contract_price.write_merged_prices_for_contract_object( futures_contract, - parquet_data, + db_data, ignore_duplication=True, ) data.log.debug( @@ -241,17 +226,17 @@ def backup_futures_contract_prices_for_contract_to_csv( # fx def backup_fx_to_csv(data): - fx_codes = data.arctic_fx_prices.get_list_of_fxcodes() + fx_codes = data.db_fx_prices.get_list_of_fxcodes() for fx_code in fx_codes: - arctic_data = data.arctic_fx_prices.get_fx_prices(fx_code) + db_data = data.db_fx_prices.get_fx_prices(fx_code) csv_data = data.csv_fx_prices.get_fx_prices(fx_code) - if check_ts_equals(arctic_data, csv_data): + if check_ts_equals(db_data, csv_data): data.log.debug("No fx backup needed for %s" % fx_code) else: # Write backup try: data.csv_fx_prices.add_fx_prices( - fx_code, arctic_data, ignore_duplication=True + fx_code, db_data, ignore_duplication=True ) data.log.debug("Written .csv backup for %s" % fx_code) except BaseException: @@ -259,24 +244,24 @@ def backup_fx_to_csv(data): def backup_multiple_to_csv(data): - instrument_list = data.arctic_futures_multiple_prices.get_list_of_instruments() + instrument_list = data.db_futures_multiple_prices.get_list_of_instruments() for instrument_code in instrument_list: backup_multiple_to_csv_for_instrument(data, instrument_code) def backup_multiple_to_csv_for_instrument(data, instrument_code: str): - arctic_data = data.arctic_futures_multiple_prices.get_multiple_prices( + db_data = data.db_futures_multiple_prices.get_multiple_prices( instrument_code ) csv_data = data.csv_futures_multiple_prices.get_multiple_prices(instrument_code) - if check_df_equals(arctic_data, csv_data): + if check_df_equals(db_data, csv_data): data.log.debug("No multiple prices backup needed for %s" % instrument_code) pass else: try: data.csv_futures_multiple_prices.add_multiple_prices( - instrument_code, arctic_data, ignore_duplication=True + instrument_code, db_data, ignore_duplication=True ) data.log.debug( "Written .csv backup multiple prices for %s" % instrument_code @@ -288,24 +273,24 @@ def backup_multiple_to_csv_for_instrument(data, instrument_code: str): def backup_adj_to_csv(data): - instrument_list = data.parquet_futures_adjusted_prices.get_list_of_instruments() + instrument_list = data.db_futures_adjusted_prices.get_list_of_instruments() for instrument_code in instrument_list: backup_adj_to_csv_for_instrument(data, instrument_code) def backup_adj_to_csv_for_instrument(data: dataBlob, instrument_code: str): - arctic_data = data.parquet_futures_adjusted_prices.get_adjusted_prices( + db_data = data.db_futures_adjusted_prices.get_adjusted_prices( instrument_code ) csv_data = data.csv_futures_adjusted_prices.get_adjusted_prices(instrument_code) - if check_ts_equals(arctic_data, csv_data): + if check_ts_equals(db_data, csv_data): data.log.debug("No adjusted prices backup needed for %s" % instrument_code) pass else: try: data.csv_futures_adjusted_prices.add_adjusted_prices( - instrument_code, arctic_data, ignore_duplication=True + instrument_code, db_data, ignore_duplication=True ) data.log.debug( "Written .csv backup for adjusted prices %s" % instrument_code @@ -317,22 +302,22 @@ def backup_adj_to_csv_for_instrument(data: dataBlob, instrument_code: str): def backup_spreads_to_csv(data: dataBlob): - instrument_list = data.arctic_spreads_for_instrument.get_list_of_instruments() + instrument_list = data.db_spreads_for_instrument.get_list_of_instruments() for instrument_code in instrument_list: backup_spreads_to_csv_for_instrument(data, instrument_code) def backup_spreads_to_csv_for_instrument(data: dataBlob, instrument_code: str): - arctic_data = data.arctic_spreads_for_instrument.get_spreads(instrument_code) + db_data = data.db_spreads_for_instrument.get_spreads(instrument_code) csv_data = data.csv_spreads_for_instrument.get_spreads(instrument_code) - if check_ts_equals(arctic_data, csv_data): + if check_ts_equals(db_data, csv_data): data.log.debug("No spreads backup needed for %s" % instrument_code) pass else: try: data.csv_spreads_for_instrument.add_spreads( - instrument_code, arctic_data, ignore_duplication=True + instrument_code, db_data, ignore_duplication=True ) data.log.debug("Written .csv backup for spreads %s" % instrument_code) except BaseException: @@ -343,24 +328,24 @@ def backup_spreads_to_csv_for_instrument(data: dataBlob, instrument_code: str): def backup_contract_position_data(data): instrument_list = ( - data.arctic_contract_position.get_list_of_instruments_with_any_position() + data.db_contract_position.get_list_of_instruments_with_any_position() ) for instrument_code in instrument_list: contract_list = ( - data.arctic_contract_position.get_list_of_contracts_for_instrument_code( + data.db_contract_position.get_list_of_contracts_for_instrument_code( instrument_code ) ) for contract in contract_list: try: - arctic_data = data.arctic_contract_position.get_position_as_series_for_contract_object( + db_data = data.db_contract_position.get_position_as_series_for_contract_object( contract ) except missingData: print("No data to write to .csv") else: data.csv_contract_position.overwrite_position_series_for_contract_object_without_checking( - contract, arctic_data + contract, db_data ) data.log.debug( "Backed up %s %s contract position data" % (instrument_code, contract) @@ -370,7 +355,7 @@ def backup_contract_position_data(data): def backup_strategy_position_data(data): strategy_list = get_list_of_strategies(data) instrument_list = ( - data.arctic_contract_position.get_list_of_instruments_with_any_position() + data.db_contract_position.get_list_of_instruments_with_any_position() ) for strategy_name in strategy_list: for instrument_code in instrument_list: @@ -378,13 +363,13 @@ def backup_strategy_position_data(data): strategy_name=strategy_name, instrument_code=instrument_code ) try: - arctic_data = data.arctic_strategy_position.get_position_as_series_for_instrument_strategy_object( + db_data = data.db_strategy_position.get_position_as_series_for_instrument_strategy_object( instrument_strategy ) except missingData: continue data.csv_strategy_position.overwrite_position_series_for_instrument_strategy_without_checking( - instrument_strategy, arctic_data + instrument_strategy, db_data ) data.log.debug( "Backed up %s %s strategy position data" @@ -395,24 +380,24 @@ def backup_strategy_position_data(data): def backup_historical_orders(data): data.log.debug("Backing up strategy orders...") list_of_orders = [ - data.mongo_strategy_historic_orders.get_order_with_orderid(id) - for id in data.mongo_strategy_historic_orders.get_list_of_order_ids() + data.db_strategy_historic_orders.get_order_with_orderid(id) + for id in data.db_strategy_historic_orders.get_list_of_order_ids() ] data.csv_strategy_historic_orders.write_orders(list_of_orders) data.log.debug("Done") data.log.debug("Backing up contract orders...") list_of_orders = [ - data.mongo_contract_historic_orders.get_order_with_orderid(order_id) - for order_id in data.mongo_contract_historic_orders.get_list_of_order_ids() + data.db_contract_historic_orders.get_order_with_orderid(order_id) + for order_id in data.db_contract_historic_orders.get_list_of_order_ids() ] data.csv_contract_historic_orders.write_orders(list_of_orders) data.log.debug("Done") data.log.debug("Backing up broker orders...") list_of_orders = [ - data.mongo_broker_historic_orders.get_order_with_orderid(order_id) - for order_id in data.mongo_broker_historic_orders.get_list_of_order_ids() + data.db_broker_historic_orders.get_order_with_orderid(order_id) + for order_id in data.db_broker_historic_orders.get_list_of_order_ids() ] data.csv_broker_historic_orders.write_orders(list_of_orders) data.log.debug("Done") @@ -434,7 +419,7 @@ def get_dict_of_strategy_capital(data: dataBlob) -> dict: for strategy_name in strategy_list: strategy_capital_data[ strategy_name - ] = data.parquet_capital.get_capital_pd_df_for_strategy(strategy_name) + ] = data.db_capital.get_capital_pd_df_for_strategy(strategy_name) return strategy_capital_data @@ -444,7 +429,7 @@ def add_total_capital_to_strategy_capital_dict_return_df( ) -> pd.DataFrame: strategy_capital_as_df = pd.concat(capital_data, axis=1) - total_capital = data.arctic_capital.get_df_of_all_global_capital() + total_capital = data.db_capital.get_df_of_all_global_capital() capital_data = pd.concat([strategy_capital_as_df, total_capital], axis=1) capital_data = capital_data.ffill() @@ -455,33 +440,33 @@ def add_total_capital_to_strategy_capital_dict_return_df( def backup_optimal_positions(data): strategy_instrument_list = ( - data.arctic_optimal_position.get_list_of_instrument_strategies_with_optimal_position() + data.db_optimal_position.get_list_of_instrument_strategies_with_optimal_position() ) for instrument_strategy in strategy_instrument_list: try: - arctic_data = data.arctic_optimal_position.get_optimal_position_as_df_for_instrument_strategy( + db_data = data.db_optimal_position.get_optimal_position_as_df_for_instrument_strategy( instrument_strategy ) except missingData: continue data.csv_optimal_position.write_optimal_position_as_df_for_instrument_strategy_without_checking( - instrument_strategy, arctic_data + instrument_strategy, db_data ) data.log.debug("Backed up %s optimal position data" % str(instrument_strategy)) def backup_spread_cost_data(data): - spread_cost_as_series = data.mongo_spread_cost.get_spread_costs_as_series() + spread_cost_as_series = data.db_spread_cost.get_spread_costs_as_series() data.csv_spread_cost.write_all_instrument_spreads(spread_cost_as_series) data.log.debug("Backed up spread cost data") def backup_roll_state_data(data): - instrument_list = data.mongo_roll_state.get_list_of_instruments() + instrument_list = data.db_roll_state.get_list_of_instruments() roll_state_list = [] for instrument_code in instrument_list: - roll_state = data.mongo_roll_state.get_name_of_roll_state(instrument_code) + roll_state = data.db_roll_state.get_name_of_roll_state(instrument_code) roll_state_list.append(roll_state) roll_state_df = pd.DataFrame(roll_state_list, index=instrument_list) @@ -492,11 +477,11 @@ def backup_roll_state_data(data): def backup_contract_data(data): instrument_list = ( - data.mongo_futures_contract.get_list_of_all_instruments_with_contracts() + data.db_futures_contract.get_list_of_all_instruments_with_contracts() ) for instrument_code in instrument_list: contract_list = ( - data.mongo_futures_contract.get_all_contract_objects_for_instrument_code( + data.db_futures_contract.get_all_contract_objects_for_instrument_code( instrument_code ) ) @@ -514,4 +499,4 @@ def backup_csv_dump(data): if __name__ == "__main__": - backup_arctic_to_csv() + backup_db_to_csv() diff --git a/sysproduction/data/capital.py b/sysproduction/data/capital.py index 7209797205..1fd4075a2a 100644 --- a/sysproduction/data/capital.py +++ b/sysproduction/data/capital.py @@ -7,18 +7,19 @@ from sysdata.production.capital import capitalData, totalCapitalCalculationData from sysdata.production.margin import marginData, seriesOfMargin -from sysdata.pointers import parquetCapitalData from sysdata.mongodb.mongo_margin import mongoMarginData from sysdata.data_blob import dataBlob from sysproduction.data.generic_production_data import productionDataLayerGeneric +from sysproduction.data.production_data_objects import get_class_for_data_type, CAPITAL_DATA from systems.accounts.from_returns import account_curve_from_returns class dataCapital(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: - data.add_class_object(parquetCapitalData) + capital_data_class = get_class_for_data_type(CAPITAL_DATA) + data.add_class_object(capital_data_class) return data diff --git a/sysproduction/data/contracts.py b/sysproduction/data/contracts.py index c7f8faa480..a08a79594f 100644 --- a/sysproduction/data/contracts.py +++ b/sysproduction/data/contracts.py @@ -1,11 +1,6 @@ import datetime -from syscore.exceptions import missingData, ContractNotFound - -from sysdata.pointers import parquetFuturesContractPriceData -from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData -from sysdata.csv.csv_roll_parameters import csvRollParametersData -from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData +from syscore.exceptions import missingData from sysdata.futures.contracts import futuresContractData from sysdata.futures.multiple_prices import futuresMultiplePricesData @@ -22,6 +17,7 @@ from sysproduction.data.prices import get_valid_instrument_code_from_user, diagPrices from sysproduction.data.generic_production_data import productionDataLayerGeneric +from sysproduction.data.production_data_objects import get_class_for_data_type, FUTURES_CONTRACT_PRICE_DATA, ROLL_PARAMETERS_DATA, FUTURES_MULTIPLE_PRICE_DATA, FUTURES_CONTRACT_DATA from sysdata.data_blob import dataBlob missing_expiry = datetime.datetime(1900, 1, 1) @@ -31,10 +27,10 @@ class dataContracts(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: data.add_class_list( [ - parquetFuturesContractPriceData, - csvRollParametersData, - arcticFuturesMultiplePricesData, - mongoFuturesContractData, + get_class_for_data_type(FUTURES_CONTRACT_PRICE_DATA), + get_class_for_data_type(ROLL_PARAMETERS_DATA), + get_class_for_data_type(FUTURES_MULTIPLE_PRICE_DATA), + get_class_for_data_type(FUTURES_CONTRACT_DATA) ] ) diff --git a/sysproduction/data/control_process.py b/sysproduction/data/control_process.py index 1959e63be3..f85964477a 100644 --- a/sysproduction/data/control_process.py +++ b/sysproduction/data/control_process.py @@ -9,12 +9,11 @@ from sysdata.config.control_config import get_control_config from sysdata.data_blob import dataBlob -from sysdata.mongodb.mongo_process_control import mongoControlProcessData from sysdata.production.process_control_data import controlProcessData from sysproduction.data.generic_production_data import productionDataLayerGeneric - +from sysproduction.data.production_data_objects import get_class_for_data_type, PROCESS_CONTROL_DATA DEFAULT_METHOD_FREQUENCY = 60 DEFAULT_MAX_EXECUTIONS = 1 DEFAULT_START_TIME_STRING = "00:01" @@ -25,7 +24,7 @@ class dataControlProcess(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: - data.add_class_object(mongoControlProcessData) + data.add_class_object(get_class_for_data_type(PROCESS_CONTROL_DATA)) return data diff --git a/sysproduction/data/currency_data.py b/sysproduction/data/currency_data.py index 0909219c95..754d998242 100644 --- a/sysproduction/data/currency_data.py +++ b/sysproduction/data/currency_data.py @@ -1,18 +1,17 @@ from syscore.constants import arg_not_supplied from syscore.interactive.menus import print_menu_of_values_and_get_response -from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData from sysdata.fx.spotfx import fxPricesData from sysdata.data_blob import dataBlob from sysobjects.spot_fx_prices import currencyValue, fxPrices from sysproduction.data.generic_production_data import productionDataLayerGeneric - +from sysproduction.data.production_data_objects import get_class_for_data_type, FX_DATA class dataCurrency(productionDataLayerGeneric): def _add_required_classes_to_data(self, data: dataBlob) -> dataBlob: - data.add_class_object(arcticFxPricesData) + data.add_class_object(get_class_for_data_type(FX_DATA)) return data @property diff --git a/sysproduction/data/instruments.py b/sysproduction/data/instruments.py index da07953908..1d9584766d 100644 --- a/sysproduction/data/instruments.py +++ b/sysproduction/data/instruments.py @@ -1,20 +1,19 @@ -from sysdata.csv.csv_instrument_data import csvFuturesInstrumentData from sysdata.data_blob import dataBlob from sysdata.futures.instruments import futuresInstrumentData from sysdata.futures.spread_costs import spreadCostData -from sysdata.mongodb.mongo_spread_costs import mongoSpreadCostData + from sysobjects.spot_fx_prices import currencyValue from sysobjects.instruments import instrumentCosts from sysproduction.data.currency_data import dataCurrency from sysproduction.data.generic_production_data import productionDataLayerGeneric from sysproduction.data.config import get_list_of_stale_instruments - +from sysproduction.data.production_data_objects import FX_DATA, SPREAD_DATA, get_class_for_data_type, FUTURES_INSTRUMENT_DATA class updateSpreadCosts(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: - data.add_class_object(mongoSpreadCostData) + data.add_class_object(get_class_for_data_type(SPREAD_DATA)) return data def update_spread_costs(self, instrument_code: str, spread_cost: float): @@ -34,7 +33,10 @@ def db_spread_cost_data(self) -> spreadCostData: class diagInstruments(productionDataLayerGeneric): def _add_required_classes_to_data(self, data: dataBlob) -> dataBlob: - data.add_class_list([csvFuturesInstrumentData, mongoSpreadCostData]) + data.add_class_list([ + get_class_for_data_type(FUTURES_INSTRUMENT_DATA), + get_class_for_data_type(SPREAD_DATA)]) + return data def get_spread_costs_as_series(self): diff --git a/sysproduction/data/optimal_positions.py b/sysproduction/data/optimal_positions.py index 60b67eb9bd..c11ec2bfbc 100644 --- a/sysproduction/data/optimal_positions.py +++ b/sysproduction/data/optimal_positions.py @@ -3,7 +3,6 @@ import pandas as pd from sysdata.data_blob import dataBlob -from sysdata.arctic.arctic_optimal_positions import arcticOptimalPositionData from sysdata.production.optimal_positions import optimalPositionData from sysobjects.production.optimal_positions import ( listOfOptimalPositionsAcrossInstrumentStrategies, @@ -18,11 +17,11 @@ get_list_of_stale_instruments, get_list_of_stale_strategies, ) - +from sysproduction.data.production_data_objects import get_class_for_data_type, OPTIMAL_POSITION_DATA class dataOptimalPositions(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: - data.add_class_object(arcticOptimalPositionData) + data.add_class_object(get_class_for_data_type(OPTIMAL_POSITION_DATA)) return data diff --git a/sysproduction/data/orders.py b/sysproduction/data/orders.py index bc7caaea10..634e130c61 100644 --- a/sysproduction/data/orders.py +++ b/sysproduction/data/orders.py @@ -2,16 +2,6 @@ from syscore.constants import arg_not_supplied from sysexecution.orders.named_order_objects import missing_order, no_parent -from sysdata.mongodb.mongo_order_stack import ( - mongoInstrumentOrderStackData, - mongoContractOrderStackData, - mongoBrokerOrderStackData, -) -from sysdata.mongodb.mongo_historic_orders import ( - mongoStrategyHistoricOrdersData, - mongoContractHistoricOrdersData, - mongoBrokerHistoricOrdersData, -) from sysdata.production.historic_orders import ( brokerHistoricOrdersData, contractHistoricOrdersData, @@ -34,6 +24,8 @@ from sysobjects.production.tradeable_object import instrumentStrategy, futuresContract +from sysproduction.data.production_data_objects import get_class_for_data_type, INSTRUMENT_ORDER_STACK_DATA, CONTRACT_ORDER_STACK_DATA, BROKER_HISTORIC_ORDERS_DATA, STRATEGY_HISTORIC_ORDERS_DATA, CONTRACT_HISTORIC_ORDERS_DATA, BROKER_ORDER_STACK_DATA + class dataOrders(object): def __init__(self, data: dataBlob = arg_not_supplied): @@ -42,12 +34,12 @@ def __init__(self, data: dataBlob = arg_not_supplied): data = dataBlob() data.add_class_list( [ - mongoInstrumentOrderStackData, - mongoContractOrderStackData, - mongoBrokerOrderStackData, - mongoContractHistoricOrdersData, - mongoStrategyHistoricOrdersData, - mongoBrokerHistoricOrdersData, + get_class_for_data_type(INSTRUMENT_ORDER_STACK_DATA), + get_class_for_data_type(CONTRACT_ORDER_STACK_DATA), + get_class_for_data_type(BROKER_ORDER_STACK_DATA), + get_class_for_data_type(STRATEGY_HISTORIC_ORDERS_DATA), + get_class_for_data_type(CONTRACT_HISTORIC_ORDERS_DATA), + get_class_for_data_type(BROKER_HISTORIC_ORDERS_DATA) ] ) self._data = data diff --git a/sysproduction/data/positions.py b/sysproduction/data/positions.py index 8e7e47a4eb..a7314373e8 100644 --- a/sysproduction/data/positions.py +++ b/sysproduction/data/positions.py @@ -7,10 +7,6 @@ from syscore.exceptions import ContractNotFound from sysexecution.orders.named_order_objects import missing_order -from sysdata.mongodb.mongo_roll_state_storage import mongoRollStateData -from sysdata.arctic.arctic_historic_contract_positions import arcticContractPositionData -from sysdata.arctic.arctic_historic_strategy_positions import arcticStrategyPositionData - from sysdata.production.roll_state import rollStateData from sysdata.production.historic_contract_positions import contractPositionData @@ -18,8 +14,6 @@ strategyPositionData, listOfInstrumentStrategyPositions, ) -from sysproduction.data.contracts import dataContracts - from sysdata.data_blob import dataBlob @@ -43,12 +37,16 @@ from sysproduction.data.generic_production_data import productionDataLayerGeneric from sysproduction.data.contracts import dataContracts - +from sysproduction.data.production_data_objects import get_class_for_data_type, ROLL_STATE_DATA, STRATEGY_POSITION_DATA, CONTRACT_POSITION_DATA class diagPositions(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: data.add_class_list( - [mongoRollStateData, arcticStrategyPositionData, arcticContractPositionData] + [ + get_class_for_data_type(ROLL_STATE_DATA), + get_class_for_data_type(STRATEGY_POSITION_DATA), + get_class_for_data_type(CONTRACT_POSITION_DATA) + ] ) return data diff --git a/sysproduction/data/prices.py b/sysproduction/data/prices.py index e5d9ac6b62..f24a3e64b3 100644 --- a/sysproduction/data/prices.py +++ b/sysproduction/data/prices.py @@ -13,20 +13,11 @@ ) from sysobjects.spreads import spreadsForInstrument -from sysdata.pointers import parquetFuturesContractPriceData from sysobjects.futures_per_contract_prices import futuresContractPrices -from sysdata.arctic.arctic_multiple_prices import ( - arcticFuturesMultiplePricesData, - futuresMultiplePrices, -) -from sysdata.pointers import parquetFuturesAdjustedPricesData +from sysobjects.multiple_prices import futuresMultiplePrices from sysobjects.adjusted_prices import futuresAdjustedPrices -from sysdata.arctic.arctic_spreads import ( - arcticSpreadsForInstrumentData, - spreadsForInstrumentData, -) -from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData +from sysdata.futures.spreads import spreadsForInstrumentData from sysdata.futures.multiple_prices import futuresMultiplePricesData from sysdata.futures.adjusted_prices import futuresAdjustedPricesData @@ -44,6 +35,7 @@ ## default for spike checking from sysproduction.data.instruments import diagInstruments, get_block_size +from sysproduction.data.production_data_objects import get_class_for_data_type, FUTURES_CONTRACT_PRICE_DATA, FUTURES_ADJUSTED_PRICE_DATA, FUTURES_MULTIPLE_PRICE_DATA, FUTURES_CONTRACT_DATA, SPREAD_DATA VERY_BIG_NUMBER = 999999.0 @@ -52,11 +44,11 @@ class diagPrices(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: data.add_class_list( [ - parquetFuturesContractPriceData, - parquetFuturesAdjustedPricesData, - arcticFuturesMultiplePricesData, - mongoFuturesContractData, - arcticSpreadsForInstrumentData, + get_class_for_data_type(FUTURES_CONTRACT_PRICE_DATA), + get_class_for_data_type(FUTURES_ADJUSTED_PRICE_DATA), + get_class_for_data_type(FUTURES_CONTRACT_DATA), + get_class_for_data_type(SPREAD_DATA), + get_class_for_data_type(FUTURES_MULTIPLE_PRICE_DATA) ] ) return data diff --git a/sysproduction/data/production_data_objects.py b/sysproduction/data/production_data_objects.py new file mode 100644 index 0000000000..c9c31b778b --- /dev/null +++ b/sysproduction/data/production_data_objects.py @@ -0,0 +1,88 @@ +from sysdata.parquet.parquet_adjusted_prices import parquetFuturesAdjustedPricesData +from sysdata.parquet.parquet_capital import parquetCapitalData +from sysdata.parquet.parquet_futures_per_contract_prices import parquetFuturesContractPriceData + +from sysdata.arctic.arctic_adjusted_prices import arcticFuturesAdjustedPricesData +from sysdata.arctic.arctic_capital import arcticCapitalData +from sysdata.arctic.arctic_futures_per_contract_prices import arcticFuturesContractPriceData +from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData +from sysdata.arctic.arctic_historic_contract_positions import arcticContractPositionData +from sysdata.arctic.arctic_historic_strategy_positions import arcticStrategyPositionData +from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData +from sysdata.arctic.arctic_optimal_positions import arcticOptimalPositionData +from sysdata.arctic.arctic_spreads import arcticSpreadsForInstrumentData + + +from sysdata.mongodb.mongo_futures_contracts import mongoFuturesContractData +from sysdata.mongodb.mongo_process_control import mongoControlProcessData +from sysdata.mongodb.mongo_order_stack import ( + mongoInstrumentOrderStackData, + mongoContractOrderStackData, + mongoBrokerOrderStackData, +) +from sysdata.mongodb.mongo_historic_orders import ( + mongoStrategyHistoricOrdersData, + mongoContractHistoricOrdersData, + mongoBrokerHistoricOrdersData, +) +from sysdata.mongodb.mongo_roll_state_storage import mongoRollStateData + + +from sysdata.csv.csv_instrument_data import csvFuturesInstrumentData +from sysdata.csv.csv_roll_parameters import csvRollParametersData + +FUTURES_CONTRACT_PRICE_DATA = "futures_contract_price_data" +FUTURES_MULTIPLE_PRICE_DATA = "futures_multiple_price_data" +FUTURES_ADJUSTED_PRICE_DATA = "futures_adjusted_price_data" +CAPITAL_DATA = "capital_data" +CONTRACT_POSITION_DATA = "contract_position_data" +STRATEGY_POSITION_DATA = "strategy_position_data" +OPTIMAL_POSITION_DATA = "optimal_position_data" +SPREAD_DATA = "spread_data" +FX_DATA = "fx_data" +ROLL_PARAMETERS_DATA = "roll_parameters_data" +FUTURES_CONTRACT_DATA = "futures_contract_data" +PROCESS_CONTROL_DATA= "process_control_data" +FUTURES_INSTRUMENT_DATA = "futures_instrument_data" +INSTRUMENT_ORDER_STACK_DATA = "instrument_order_stack_data" +CONTRACT_ORDER_STACK_DATA = "contract_order_stack_data" +BROKER_ORDER_STACK_DATA = "broker_order_stack_data" +STRATEGY_HISTORIC_ORDERS_DATA = "strategy_historic_orders_data" +CONTRACT_HISTORIC_ORDERS_DATA = "contract_historic_orders_data" +BROKER_HISTORIC_ORDERS_DATA = "broker_historic_orders_data" +ROLL_STATE_DATA = "roll_state_data" + +use_production_classes = { + FX_DATA: arcticFxPricesData, + ROLL_PARAMETERS_DATA: csvRollParametersData, + FUTURES_INSTRUMENT_DATA: csvFuturesInstrumentData, + FUTURES_CONTRACT_DATA: mongoFuturesContractData, + + FUTURES_CONTRACT_PRICE_DATA: parquetFuturesContractPriceData, + FUTURES_MULTIPLE_PRICE_DATA: arcticFuturesMultiplePricesData, + FUTURES_ADJUSTED_PRICE_DATA: parquetFuturesAdjustedPricesData, + + CAPITAL_DATA: parquetCapitalData, + + CONTRACT_POSITION_DATA: arcticContractPositionData, + STRATEGY_POSITION_DATA: arcticStrategyPositionData, + OPTIMAL_POSITION_DATA: arcticOptimalPositionData, + SPREAD_DATA: arcticSpreadsForInstrumentData, + + STRATEGY_HISTORIC_ORDERS_DATA: mongoStrategyHistoricOrdersData, + CONTRACT_HISTORIC_ORDERS_DATA: mongoContractHistoricOrdersData, + BROKER_HISTORIC_ORDERS_DATA: mongoBrokerHistoricOrdersData, + + INSTRUMENT_ORDER_STACK_DATA: mongoInstrumentOrderStackData, + CONTRACT_ORDER_STACK_DATA: mongoContractOrderStackData, + BROKER_HISTORIC_ORDERS_DATA: mongoBrokerOrderStackData, + + ROLL_STATE_DATA: mongoRollStateData, + + PROCESS_CONTROL_DATA: mongoControlProcessData +} + +def get_class_for_data_type(data_type:str): + + return use_production_classes[data_type] + diff --git a/sysproduction/data/sim_data.py b/sysproduction/data/sim_data.py index bcbd921a1c..2eb3c622cb 100644 --- a/sysproduction/data/sim_data.py +++ b/sysproduction/data/sim_data.py @@ -2,13 +2,8 @@ from sysdata.sim.db_futures_sim_data import dbFuturesSimData from sysdata.data_blob import dataBlob -from sysdata.parquet.parquet_adjusted_prices import parquetFuturesAdjustedPricesData -from sysdata.arctic.arctic_multiple_prices import arcticFuturesMultiplePricesData -from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData -from sysdata.csv.csv_instrument_data import csvFuturesInstrumentData -from sysdata.mongodb.mongo_spread_costs import mongoSpreadCostData -from sysdata.csv.csv_roll_parameters import csvRollParametersData +from sysproduction.data.production_data_objects import get_class_for_data_type, FUTURES_ADJUSTED_PRICE_DATA, FUTURES_MULTIPLE_PRICE_DATA, FX_DATA, SPREAD_DATA, FUTURES_INSTRUMENT_DATA, ROLL_PARAMETERS_DATA def get_sim_data_object_for_production(data=arg_not_supplied) -> dbFuturesSimData: # Check data has the right elements to do this @@ -17,12 +12,12 @@ def get_sim_data_object_for_production(data=arg_not_supplied) -> dbFuturesSimDat data.add_class_list( [ - parquetFuturesAdjustedPricesData, - arcticFuturesMultiplePricesData, - arcticFxPricesData, - mongoSpreadCostData, - csvFuturesInstrumentData, - csvRollParametersData, + get_class_for_data_type(FUTURES_ADJUSTED_PRICE_DATA), + get_class_for_data_type(FUTURES_MULTIPLE_PRICE_DATA), + get_class_for_data_type(FX_DATA), + get_class_for_data_type(SPREAD_DATA), + get_class_for_data_type(FUTURES_INSTRUMENT_DATA), + get_class_for_data_type(ROLL_PARAMETERS_DATA) ] ) diff --git a/sysproduction/data/volumes.py b/sysproduction/data/volumes.py index 1a5cf11a20..532daa7872 100644 --- a/sysproduction/data/volumes.py +++ b/sysproduction/data/volumes.py @@ -1,12 +1,12 @@ import datetime as datetime import pandas as pd from syscore.exceptions import missingData -from sysdata.pointers import parquetFuturesContractPriceData from sysdata.futures.futures_per_contract_prices import futuresContractPriceData from sysobjects.contracts import futuresContract from sysdata.data_blob import dataBlob from sysproduction.data.generic_production_data import productionDataLayerGeneric +from sysproduction.data.production_data_objects import FUTURES_CONTRACT_PRICE_DATA, get_class_for_data_type # Get volume data for the contract we're currently trading, plus what we might roll into, plus the previous one # This is handy for working out whether to roll @@ -16,7 +16,7 @@ class diagVolumes(productionDataLayerGeneric): def _add_required_classes_to_data(self, data) -> dataBlob: - data.add_class_object(parquetFuturesContractPriceData) + data.add_class_object(get_class_for_data_type(FUTURES_CONTRACT_PRICE_DATA)) return data @property