adding more untested execution code

docs/production.md

@@ -484,8 +484,9 @@ The following should be used as logging attributes (failure to do so will break
 - component: other parts of the top level function that have their own loggers
 - currency_code: Currency code (used for fx), format 'GBPUSD'
 - instrument_code: Self explanatory
-- contract_date: Self explanatory, format 'yyyymm' 
-- order_id: Self explanatory, used for live trading
+- contract_date: Self explanatory, format 'yyyymmdd' 
+- instrument_order_id, contract_order_id, broker_order_id: Self explanatory, used for live trading
+- strategy_name: Self explanatory
 
 
 ### Getting log data back

sysbrokers/IB/ibClient.py

@@ -413,9 +413,9 @@ def _unique_list_from_total(account_summary_data, tag_name):
 
 def get_barsize_and_duration_from_frequency(bar_freq):
 
-    barsize_lookup = dict([('D', "1 day"), ('H', "1 hour"), ('5M', '5 mins'), ('M', '1 min'),
+    barsize_lookup = dict([('D', "1 day"), ('H', "1 hour"), ('15M', '15 mins'), ('5M', '5 mins'), ('M', '1 min'),
                            ('10S', '10 secs'), ('S', '1 secs')])
-    duration_lookup = dict([('D', "1 Y"), ('H', "1 M"), ('5M', "1 W"), ('M', '1 D'),
+    duration_lookup = dict([('D', "1 Y"), ('H', "1 M"), ('15M', '1 W'), ('5M', "1 W"), ('M', '1 D'),
                             ('10S', '14400 S'), ('S', '1800 S')])
     try:
         assert bar_freq in barsize_lookup.keys() and bar_freq in duration_lookup.keys()

sysbrokers/IB/ibFuturesContractPriceData.py

@@ -1,8 +1,10 @@
 
 import pandas as pd
-
+import datetime
 from syscore.fileutils import get_filename_for_package
 from syscore.genutils import value_or_npnan, NOT_REQUIRED
+
+
 from sysdata.futures.futures_per_contract_prices import futuresContractPriceData, futuresContractPrices
 from sysdata.futures.contracts import futuresContract, listOfFuturesContracts
 from sysdata.futures.instruments import futuresInstrument
@@ -131,6 +133,9 @@ def get_prices_for_contract_object(self, contract_object):
         else:
             data = futuresContractPrices(price_data)
 
+        data = futuresContractPrices(data[data.index<datetime.datetime.now()])
+        data = data.remove_zero_volumes()
+
         return data
 
     def get_prices_at_frequency_for_contract_object(self, contract_object, freq="D"):
@@ -141,7 +146,7 @@ def get_prices_at_frequency_for_contract_object(self, contract_object, freq="D")
         Because the list of dates returned by contracts_with_price_data is likely to not match (expiries)
 
         :param contract_object:  futuresContract
-        :param freq: str; one of D, H, 5M, M, 10S, S
+        :param freq: str; one of D, H, 15M, 5M, M, 10S, S
         :return: data
         """
         new_log = self.log.setup(instrument_code=contract_object.instrument_code, contract_date=contract_object.date)
@@ -158,7 +163,10 @@ def get_prices_at_frequency_for_contract_object(self, contract_object, freq="D")
             new_log.warn("No IB price data found for %s" % str(contract_object))
             data = futuresContractPrices.create_empty()
         else:
-            data = futuresContractPrices.only_have_final_prices(price_data)
+            data = futuresContractPrices(price_data)
+
+        data = futuresContractPrices(data[data.index<datetime.datetime.now()])
+        data = data.remove_zero_volumes()
 
         return data
 

syscore/dateutils.py

@@ -257,6 +257,7 @@ def long_to_datetime(float_to_convert):
     converted_datetime = datetime.datetime.strptime(str_to_convert, LONG_DATE_FORMAT)
     return converted_datetime
 
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod()

syscore/genutils.py

@@ -276,6 +276,20 @@ def check_if_ready_for_another_run(self):
         else:
             return False
 
+# avoids encoding problems with mongo
+_none = ""
+
+def none_to_object(x, object):
+    if x is _none:
+        return object
+    else:
+        return x
+
+def object_to_none(x, object, y=_none):
+    if x is object:
+        return y
+    else:
+        return x
 
 
 if __name__ == '__main__':

syscore/objects.py

@@ -19,6 +19,18 @@ def __repr__(self):
 locked_order = _named_object("locked order")
 duplicate_order = _named_object("duplicate order")
 
+order_is_in_status_finished = _named_object("order status is modification finished")
+order_is_in_status_modified = _named_object("order status is being modified")
+order_is_in_status_not_modified = _named_object("order status is not currently modified")
+order_is_in_status_reject_modification= _named_object("order status is modification rejected")
+
+
+no_order_id = _named_object("no order ID")
+no_children = _named_object("no_children")
+no_parent = _named_object("no parent")
+
+rolling_cant_trade = _named_object("rolling can't trade")
+ROLL_PSEUDO_STRATEGY="_ROLL_PSEUDO_STRATEGY"
 
 data_error = _named_object("data error")
 not_updated = _named_object("not updated")

syscore/pdutils.py

@@ -414,7 +414,7 @@ def _first_spike_in_data(merged_data, first_date_in_new_data=None, column_to_che
     ## absolute is what matters
     abs_change_pd = change_pd.abs()
     ## hard to know what span to use here as could be daily, intraday or a mixture
-    avg_abs_change = abs_change_pd.ewm(span=250).mean()
+    avg_abs_change = abs_change_pd.ewm(span=500).mean()
 
     change_in_avg_units = abs_change_pd / avg_abs_change
 

sysdata/futures/futures_per_contract_prices.py

@@ -108,6 +108,10 @@ def _full_merge_of_existing_data(self, new_futures_per_contract_prices):
 
         return futuresContractPrices(merged_data)
 
+    def remove_zero_volumes(self):
+        self[self[VOLUME_COLUMN] > 0]
+        return futuresContractPrices(self)
+
     def add_rows_to_existing_data(self, new_futures_per_contract_prices, check_for_spike=True):
         """
         Merges self with new data.

sysdata/mongodb/mongo_historic_orders.py

@@ -0,0 +1,129 @@
+from syscore.objects import success, missing_order, resolve_function
+from sysdata.mongodb.mongo_connection import mongoConnection, MONGO_ID_KEY
+from syslogdiag.log import logtoscreen
+from sysdata.production.historic_orders import genericOrdersData, strategyHistoricOrdersData, contractHistoricOrdersData
+
+ORDER_ID_STORE_KEY = "_ORDER_ID_STORE_KEY"
+
+class mongoGenericHistoricOrdersData(genericOrdersData):
+    """
+    Read and write data class to get roll state data
+
+
+    """
+    def _collection_name(self):
+        raise NotImplementedError("Need to inherit for a specific data type")
+
+    def _order_class_str(self):
+        raise NotImplementedError("Need to inherit for a specific data type")
+
+    def _order_class(self):
+        class_as_str = self._order_class_str()
+        return resolve_function(class_as_str)
+
+
+    def __init__(self, mongo_db = None, log=logtoscreen("mongoGenericHistoricOrdersData")):
+        # Not needed as we don't store anything in _state attribute used in parent class
+        # If we did have _state would risk breaking if we forgot to override methods
+        #super().__init__()
+
+        self._mongo = mongoConnection(self._collection_name(), mongo_db=mongo_db)
+
+        # this won't create the index if it already exists
+        self._mongo.create_index("order_id")
+        self.log = log
+
+    @property
+    def _name(self):
+        return "Generic historic orders"
+
+    def __repr__(self):
+        return "Data connection for %s, mongodb %s/%s @ %s -p %s " % (self._name,
+            self._mongo.database_name, self._mongo.collection_name, self._mongo.host, self._mongo.port)
+
+    def add_order_to_data(self, order):
+        # Doesn't check for duplicates
+        order_id = self._get_next_order_id()
+        order.order_id = order_id
+        mongo_record = order.as_dict()
+        self._mongo.collection.insert_one(mongo_record)
+        return success
+
+
+    def get_order_with_orderid(self, order_id):
+        result_dict = self._mongo.collection.find_one(dict(order_id = order_id))
+        if result_dict is None:
+            return missing_order
+        result_dict.pop(MONGO_ID_KEY)
+
+        order_class = self._order_class()
+        order = order_class.from_dict(result_dict)
+        return order
+
+    def delete_order_with_orderid(self, order_id):
+        pass
+
+    def update_order_with_orderid(self, order_id, order):
+        self._mongo.collection.update_one(dict(order_id=order_id), {'$set': order.as_dict()})
+
+
+    def get_list_of_order_ids(self):
+        cursor = self._mongo.collection.find()
+        order_ids = [db_entry['order_id'] for db_entry in cursor]
+        order_ids.remove(ORDER_ID_STORE_KEY)
+
+        return order_ids
+
+
+
+    # ORDER ID
+    def _get_next_order_id(self):
+        max_orderid = self._get_current_max_order_id()
+        new_orderid = max_orderid + 1
+        self._update_max_order_id(new_orderid)
+
+        return new_orderid
+
+    def _get_current_max_order_id(self):
+        result_dict = self._mongo.collection.find_one(dict(order_id=ORDER_ID_STORE_KEY))
+        if result_dict is None:
+            return self._create_max_order_id()
+
+        result_dict.pop(MONGO_ID_KEY)
+        order_id = result_dict['max_order_id']
+
+        return order_id
+
+    def _update_max_order_id(self, max_order_id):
+        self._mongo.collection.update_one(dict(order_id=ORDER_ID_STORE_KEY), {'$set': dict(max_order_id=max_order_id)})
+
+        return success
+
+    def _create_max_order_id(self):
+        first_order_id = 1
+        self._mongo.collection.insert_one(dict(order_id=ORDER_ID_STORE_KEY, max_order_id=first_order_id))
+        return first_order_id
+
+
+class mongoStrategyHistoricOrdersData(mongoGenericHistoricOrdersData, strategyHistoricOrdersData):
+    def _collection_name(self):
+        return "_STRATEGY_HISTORIC_ORDERS"
+
+    def _order_class_str(self):
+        return "sysdata.production.historic_orders.historicStrategyOrder"
+
+    def get_list_of_orders_for_strategy(self, strategy_name):
+        raise NotImplementedError
+
+    def get_list_of_orders_for_strategy_and_instrument(self, strategy_name, instrument_code):
+        raise NotImplementedError
+
+class mongoContractHistoricOrdersData(mongoGenericHistoricOrdersData, contractHistoricOrdersData):
+    def _collection_name(self):
+        return "_CONTRACT_HISTORIC_ORDERS"
+
+    def _order_class_str(self):
+        return "sysdata.production.historic_orders.historicContractOrder"
+
+    def get_list_of_orders_since_date(self, recent_datetime):
+        raise NotImplementedError