added position data to backup

sysdata/parquet/parquet_historic_contract_positions.py

@@ -0,0 +1,64 @@
+
+import pandas as pd
+
+from sysobjects.contracts import futuresContract, listOfFuturesContracts
+from sysdata.parquet.parquet_access import ParquetAccess
+
+from sysdata.production.historic_contract_positions import contractPositionData
+
+from syscore.exceptions import missingData
+
+from syslogging.logger import *
+
+CONTRACT_POSITION_COLLECTION = "contract_positions"
+
+
+class parquetContractPositionData(contractPositionData):
+    def __init__(self, parquet_access: ParquetAccess,  log=get_logger("parquetContractPositionData")):
+
+        super().__init__(log=log)
+
+        self._parquet = parquet_access
+
+    def __repr__(self):
+        return "parquetContractPositionData"
+
+    @property
+    def parquet(self):
+        return self._parquet
+
+    def _write_updated_position_series_for_contract_object(
+        self, contract_object: futuresContract, updated_series: pd.Series
+    ):
+        ## overwrites what is there without checking
+        ident = contract_object.key
+        updated_data_as_df = pd.DataFrame(updated_series)
+        updated_data_as_df.columns = ["position"]
+
+        self.parquet.write_data_given_data_type_and_identifier(data_to_write=updated_data_as_df, identifier=ident, data_type=CONTRACT_POSITION_COLLECTION)
+
+    def _delete_position_series_for_contract_object_without_checking(
+        self, contract_object: futuresContract
+    ):
+        ident = contract_object.key
+        self.parquet.delete_data_given_data_type_and_identifier(data_type=CONTRACT_POSITION_COLLECTION, identifier=ident)
+
+    def get_position_as_series_for_contract_object(
+        self, contract_object: futuresContract
+    ) -> pd.Series:
+        keyname = contract_object.key
+        try:
+            pd_df = self.parquet.read_data_given_data_type_and_identifier(data_type=CONTRACT_POSITION_COLLECTION, identifier=keyname)
+        except:
+            raise missingData
+
+        return pd_df.iloc[:, 0]
+
+    def get_list_of_contracts(self) -> listOfFuturesContracts:
+        ## doesn't remove zero positions
+        list_of_keys = self.parquet.get_all_identifiers_with_data_type(data_type=CONTRACT_POSITION_COLLECTION)
+        list_of_futures_contract = [
+            futuresContract.from_key(key) for key in list_of_keys
+        ]
+
+        return listOfFuturesContracts(list_of_futures_contract)

sysdata/parquet/parquet_historic_strategy_positions.py

@@ -0,0 +1,65 @@
+import pandas as pd
+
+from sysobjects.production.tradeable_object import (
+    listOfInstrumentStrategies,
+    instrumentStrategy,
+)
+from sysdata.parquet.parquet_access import ParquetAccess
+from sysdata.production.historic_strategy_positions import strategyPositionData
+from syscore.exceptions import missingData
+
+from syslogging.logger import *
+
+STRATEGY_POSITION_COLLECTION = "strategy_positions"
+
+
+class parquetStrategyPositionData(strategyPositionData):
+    def __init__(self, parquet_access: ParquetAccess, log=get_logger("parquetStrategyPositionData")):
+
+        super().__init__(log=log)
+
+        self._parquet = parquet_access
+
+    def __repr__(self):
+        return "parquetStrategyPositionData"
+
+    @property
+    @property
+    def parquet(self):
+        return self._parquet
+
+    def get_list_of_instrument_strategies(self) -> listOfInstrumentStrategies:
+        list_of_keys = self.parquet.get_all_identifiers_with_data_type(data_type=STRATEGY_POSITION_COLLECTION)
+        list_of_instrument_strategies = [
+            instrumentStrategy.from_key(key) for key in list_of_keys
+        ]
+
+        return listOfInstrumentStrategies(list_of_instrument_strategies)
+
+    def _write_updated_position_series_for_instrument_strategy_object(
+        self, instrument_strategy: instrumentStrategy, updated_series: pd.Series
+    ):
+
+        ident = instrument_strategy.key
+        updated_data_as_df = pd.DataFrame(updated_series)
+        updated_data_as_df.columns = ["position"]
+
+        self.parquet.write_data_given_data_type_and_identifier(data_to_write=updated_data_as_df, identifier=ident, data_type=STRATEGY_POSITION_COLLECTION)
+
+    def _delete_position_series_for_instrument_strategy_object_without_checking(
+        self, instrument_strategy: instrumentStrategy
+    ):
+        ident = instrument_strategy.key
+        self.parquet.delete_data_given_data_type_and_identifier(data_type=STRATEGY_POSITION_COLLECTION, identifier=ident)
+
+    def get_position_as_series_for_instrument_strategy_object(
+        self, instrument_strategy: instrumentStrategy
+    ) -> pd.Series:
+
+        keyname = instrument_strategy.key
+        try:
+            pd_df = self.parquet.read_data_given_data_type_and_identifier(data_type=STRATEGY_POSITION_COLLECTION, identifier=keyname)
+        except:
+            raise missingData
+
+        return pd_df.iloc[:, 0]

sysinit/transfer/backup_arctic_to_parquet.py

@@ -16,6 +16,8 @@
 from sysdata.parquet.parquet_spotfx_prices import parquetFxPricesData
 from sysdata.parquet.parquet_spreads import parquetSpreadsForInstrumentData
 from sysdata.parquet.parquet_optimal_positions import parquetOptimalPositionData
+from sysdata.parquet.parquet_historic_contract_positions import parquetContractPositionData
+from sysdata.parquet.parquet_historic_strategy_positions import parquetStrategyPositionData
 
 from sysdata.csv.csv_futures_contracts import csvFuturesContractData
 from sysdata.csv.csv_contract_position_data import csvContractPositionData
@@ -76,19 +78,25 @@ def backup_arctic_to_parquet():
         do = true_if_answer_is_yes("Adjusted prices?")
         if do:
             backup_adj_to_parquet(backup_data)
-        #backup_strategy_position_data(backup_data)
-        #backup_contract_position_data(backup_data)
-        #backup_historical_orders(backup_data)
+        do = true_if_answer_is_yes("Strategy positions?")
+        if do:
+            backup_strategy_position_data(backup_data)
+        do = true_if_answer_is_yes("Contract positions?")
+        if do:
+            backup_contract_position_data(backup_data)
+
         do = true_if_answer_is_yes("Capital?")
         if do:
             backup_capital(backup_data)
-        #backup_contract_data(backup_data)
         do = true_if_answer_is_yes("Time series of spread costs?")
         if do:
             backup_spreads_to_parquet(backup_data)
         do = true_if_answer_is_yes("optimal positions?")
         if do:
             backup_optimal_positions(backup_data)
+
+        # backup_contract_data(backup_data)
+        # backup_historical_orders(backup_data)
         #backup_roll_state_data(backup_data)
 
 
@@ -343,12 +351,28 @@ def backup_contract_position_data(data):
                 )
             except missingData:
                 print("No data to write to .csv")
-            else:
-                data.csv_contract_position.overwrite_position_series_for_contract_object_without_checking(
-                    contract, arctic_data
+                continue
+
+            try:
+                parquet_data = data.parquet_contract_position.get_position_as_series_for_contract_object(
+                    contract
                 )
+            except missingData:
+                parquet_data = []
+
+            if len(parquet_data)>=len(arctic_data):
+                data.log.debug("Skipping")
+                continue
+
+            data.parquet_contract_position.overwrite_position_series_for_contract_object_without_checking(
+                contract, arctic_data
+            )
+            parquet_data = data.parquet_contract_position.get_position_as_series_for_contract_object(
+                contract
+            )
+
             data.log.debug(
-                "Backed up %s %s contract position data" % (instrument_code, contract)
+                "Backed up %s %s contract position data was %s now %s" % (instrument_code, contract, str(arctic_data), str(parquet_data))
             )
 
 
@@ -368,12 +392,28 @@ def backup_strategy_position_data(data):
                 )
             except missingData:
                 continue
-            data.csv_strategy_position.overwrite_position_series_for_instrument_strategy_without_checking(
+
+            try:
+                parquet_data = data.parquet_strategy_position.get_position_as_series_for_instrument_strategy_object(
+                    instrument_strategy
+                )
+            except missingData:
+                parquet_data = []
+            if len(parquet_data)>=len(arctic_data):
+                data.log.debug("Skipping")
+                continue
+
+            data.parquet_strategy_position.overwrite_position_series_for_instrument_strategy_without_checking(
                 instrument_strategy, arctic_data
             )
+
+            parquet_data = data.parquet_strategy_position.get_position_as_series_for_instrument_strategy_object(
+                instrument_strategy
+            )
+
             data.log.debug(
-                "Backed up %s %s strategy position data"
-                % (instrument_code, strategy_name)
+                "Backed up %s %s strategy position data was %s now %s"
+                % (instrument_code, strategy_name, str(arctic_data), str(parquet_data))
             )