Merge branch 'develop' into python_logging_switch

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release notes
 
+## Version 1.62
+
+- Added order simulator as an optimal replacement for vectorised p&l calculation; prequisite for limit order simulation
+- Replace pst logging with python logging
+- ignore daily expiries for certain EUREX contracts
+- Allow fixed instrument and forecast weights to be specificed as a hierarchy
+
 ## Version 1.61
 
 - Replaced log to database with log to file

README.md

@@ -7,10 +7,10 @@ Rob Carver
 [https://qoppac.blogspot.com/p/pysystemtrade.html](https://qoppac.blogspot.com/p/pysystemtrade.html)
 
 
-Version 1.61
+Version 1.62
 
 
-2023-03-24
+2023-06-09
 
 
 

docs/backtesting.md

@@ -4399,7 +4399,7 @@ Other methods exist to access logging and caching.
 | `positionSize.get_block_value` | Standard | `instrument_code` | D | Get value of a 1% move in the price |
 | `positionSize.get_instrument_currency_vol` | Standard | `instrument_code` |D | Get daily volatility in the currency of the instrument |
 | `positionSize.get_instrument_value_vol` | Standard | `instrument_code` |D | Get daily volatility in the currency of the trading account |
-| `positionSize.get_volatility_scalar` | Standard | `instrument_code` | D |Get ratio of target volatility vs volatility of instrument in instrument's own currency |
+| `positionSize.get_average_position_at_subsystem_level` | Standard | `instrument_code` | D |Get ratio of target volatility vs volatility of instrument in instrument's own currency |
 | `positionSize.get_subsystem_position`| Standard | `instrument_code` | D, O |Get position if we put our entire trading capital into one instrument |
 
 

examples/introduction/simplesystem.py

@@ -150,7 +150,7 @@
 print(my_system.positionSize.get_block_value("SOFR").tail(5))
 print(my_system.positionSize.get_underlying_price("SOFR"))
 print(my_system.positionSize.get_instrument_value_vol("SOFR").tail(5))
-print(my_system.positionSize.get_volatility_scalar("SOFR").tail(5))
+print(my_system.positionSize.get_average_position_at_subsystem_level("SOFR").tail(5))
 print(my_system.positionSize.get_vol_target_dict())
 print(my_system.positionSize.get_subsystem_position("SOFR").tail(5))
 

sysbrokers/IB/config/ib_fx_config.py

@@ -2,7 +2,8 @@
 
 import pandas as pd
 
-from syscore.constants import missing_file, missing_instrument
+from syscore.constants import missing_file
+from syscore.exceptions import missingInstrument
 from syscore.fileutils import resolve_path_and_filename_for_package
 from syslogging.logger import *
 
@@ -31,7 +32,7 @@ def config_info_for_code(config_data: pd.DataFrame, currency_code, log) -> ibFXC
             "Can't get IB FX config for %s as config file missing" % currency_code
         )
 
-        return missing_instrument
+        raise missingInstrument
 
     ccy1 = config_data[config_data.CODE == currency_code].CCY1.values[0]
     ccy2 = config_data[config_data.CODE == currency_code].CCY2.values[0]

sysbrokers/IB/ib_Fx_prices_data.py

@@ -8,11 +8,10 @@
     ibFXConfig,
 )
 from sysbrokers.broker_fx_prices_data import brokerFxPricesData
-from syscore.exceptions import missingData
+from syscore.exceptions import missingData, missingInstrument
 from sysdata.data_blob import dataBlob
 from sysobjects.spot_fx_prices import fxPrices
 from syslogging.logger import *
-from syscore.constants import missing_instrument
 
 
 class ibFxPricesData(brokerFxPricesData):
@@ -45,9 +44,9 @@ def get_list_of_fxcodes(self) -> list:
         return list_of_codes
 
     def _get_fx_prices_without_checking(self, currency_code: str) -> fxPrices:
-        ib_config_for_code = self._get_config_info_for_code(currency_code)
-
-        if ib_config_for_code is missing_instrument:
+        try:
+            ib_config_for_code = self._get_config_info_for_code(currency_code)
+        except missingInstrument:
             log = self.log.setup(**{CURRENCY_CODE_LOG_LABEL: currency_code})
             log.warn("Can't get prices as missing IB config for %s" % currency_code)
             return fxPrices.create_empty()

syscore/constants.py

@@ -9,7 +9,6 @@ def __repr__(self):
         return self._name
 
 
-missing_instrument = named_object("missing instrument")
 missing_file = named_object("missing file")
 market_closed = named_object("market closed")
 fill_exceeds_trade = named_object("fill too big for trade")

syscore/genutils.py

@@ -132,6 +132,10 @@ def str_of_int(x: int) -> str:
         return ""
 
 
+def same_sign(x, y):
+    return sign(x) == sign(y)
+
+
 def sign(x: Union[int, float]) -> float:
     """
     >>> sign(3)

sysdata/production/historic_orders.py

@@ -19,7 +19,7 @@
 from sysexecution.orders.named_order_objects import missing_order
 
 from sysdata.base_data import baseData
-from sysobjects.fills import listOfFills, fill_from_order
+from sysobjects.fills import ListOfFills, fill_from_order
 from sysexecution.orders.base_orders import Order
 from sysexecution.orders.broker_orders import single_fill_from_broker_order
 from sysexecution.order_stacks.order_stack import missingOrder
@@ -71,7 +71,7 @@ def get_list_of_order_ids_in_date_range(
 class strategyHistoricOrdersData(genericOrdersData):
     def get_fills_history_for_instrument_strategy(
         self, instrument_strategy: instrumentStrategy
-    ) -> listOfFills:
+    ) -> ListOfFills:
         """
 
         :param instrument_code:  str
@@ -82,7 +82,7 @@ def get_fills_history_for_instrument_strategy(
             instrument_strategy
         )
         order_list_as_fills = [fill_from_order(order) for order in order_list]
-        list_of_fills = listOfFills(order_list_as_fills)
+        list_of_fills = ListOfFills(order_list_as_fills)
 
         return list_of_fills
 
@@ -115,7 +115,7 @@ class contractHistoricOrdersData(genericOrdersData):
 class brokerHistoricOrdersData(contractHistoricOrdersData):
     def get_fills_history_for_contract(
         self, futures_contract: futuresContract
-    ) -> listOfFills:
+    ) -> ListOfFills:
         """
 
         :param instrument_code:  str
@@ -133,7 +133,7 @@ def get_fills_history_for_contract(
             for orderid in list_of_order_ids
         ]
         list_of_fills = [fill for fill in list_of_fills if fill is not missing_order]
-        list_of_fills = listOfFills(list_of_fills)
+        list_of_fills = ListOfFills(list_of_fills)
 
         return list_of_fills
 

sysexecution/orders/named_order_objects.py

@@ -1,5 +1,6 @@
 from syscore.constants import named_object
 
+not_filled = named_object("not filled")
 missing_order = named_object("missing order")
 locked_order = named_object("locked order")
 duplicate_order = named_object("duplicate order")

sysobjects/fills.py

@@ -1,23 +1,35 @@
+from typing import Union
 import datetime
+from dataclasses import dataclass
 from collections import namedtuple
 
 import pandas as pd
 
 from syscore.constants import named_object
-from sysexecution.orders.named_order_objects import missing_order
-from sysobjects.orders import SimpleOrder, ListOfSimpleOrders
+from sysexecution.orders.named_order_objects import missing_order, not_filled
+from sysobjects.orders import (
+    SimpleOrder,
+    ListOfSimpleOrders,
+    ListOfSimpleOrdersWithDate,
+    SimpleOrderWithDate,
+)
 
-from sysexecution.orders.list_of_orders import listOfOrders
 from sysexecution.orders.base_orders import Order
 
-Fill = namedtuple("Fill", ["date", "qty", "price"])
 
-NOT_FILLED = named_object("not filled")
+@dataclass
+class Fill:
+    date: datetime.datetime
+    qty: int
+    price: float
+    includes_slippage: bool = False
 
 
-class listOfFills(list):
+class ListOfFills(list):
     def __init__(self, list_of_fills):
-        list_of_fills = [fill for fill in list_of_fills if fill is not missing_order]
+        list_of_fills = [
+            fill for fill in list_of_fills if fill is not (missing_order or not_filled)
+        ]
         super().__init__(list_of_fills)
 
     def _as_dict_of_lists(self) -> dict:
@@ -47,7 +59,7 @@ def from_position_series_and_prices(cls, positions: pd.Series, price: pd.Series)
 
 def _list_of_fills_from_position_series_and_prices(
     positions: pd.Series, price: pd.Series
-) -> listOfFills:
+) -> ListOfFills:
 
     (
         trades_without_zeros,
@@ -63,7 +75,7 @@ def _list_of_fills_from_position_series_and_prices(
         for date, qty, price in zip(dates_as_list, trades_as_list, prices_as_list)
     ]
 
-    list_of_fills = listOfFills(list_of_fills_as_list)
+    list_of_fills = ListOfFills(list_of_fills_as_list)
 
     return list_of_fills
 
@@ -103,20 +115,44 @@ def fill_from_order(order: Order) -> Fill:
     return Fill(fill_datetime, fill_qty, fill_price)
 
 
+def fill_list_of_simple_orders(
+    list_of_orders: ListOfSimpleOrders,
+    fill_datetime: datetime.datetime,
+    market_price: float,
+) -> Fill:
+    list_of_fills = [
+        fill_from_simple_order(
+            simple_order=simple_order,
+            fill_datetime=fill_datetime,
+            market_price=market_price,
+        )
+        for simple_order in list_of_orders
+    ]
+    list_of_fills = ListOfFills(list_of_fills)  ## will remove unfilled
+
+    if len(list_of_fills) == 0:
+        return not_filled
+    elif len(list_of_fills) == 1:
+        return list_of_fills[0]
+    else:
+        raise Exception(
+            "List of orders %s has produced more than one fill %s!"
+            % (str(list_of_orders), str(list_of_orders))
+        )
+
+
 def fill_from_simple_order(
     simple_order: SimpleOrder,
     market_price: float,
     fill_datetime: datetime.datetime,
-    slippage: float = 0,
 ) -> Fill:
     if simple_order.is_zero_order:
-        return NOT_FILLED
+        return not_filled
 
     elif simple_order.is_market_order:
         fill = fill_from_simple_market_order(
             simple_order,
             market_price=market_price,
-            slippage=slippage,
             fill_datetime=fill_datetime,
         )
     else:
@@ -129,31 +165,39 @@ def fill_from_simple_order(
 
 
 def fill_from_simple_limit_order(
-    simple_order: SimpleOrder, market_price: float, fill_datetime: datetime.datetime
+    simple_order: Union[SimpleOrder, SimpleOrderWithDate],
+    market_price: float,
+    fill_datetime: datetime.datetime,
 ) -> Fill:
 
     limit_price = simple_order.limit_price
     if simple_order.quantity > 0:
         if limit_price > market_price:
-            return Fill(fill_datetime, simple_order.quantity, limit_price)
+            return Fill(
+                fill_datetime,
+                simple_order.quantity,
+                limit_price,
+                includes_slippage=True,
+            )
 
     if simple_order.quantity < 0:
         if limit_price < market_price:
-            return Fill(fill_datetime, simple_order.quantity, limit_price)
+            return Fill(
+                fill_datetime,
+                simple_order.quantity,
+                limit_price,
+                includes_slippage=True,
+            )
 
-    return NOT_FILLED
+    return not_filled
 
 
 def fill_from_simple_market_order(
-    simple_order: SimpleOrder,
+    simple_order: Union[SimpleOrder, SimpleOrderWithDate],
     market_price: float,
     fill_datetime: datetime.datetime,
-    slippage: float = 0,
 ) -> Fill:
 
-    if simple_order.quantity > 0:
-        fill_price_with_slippage = market_price + slippage
-    else:
-        fill_price_with_slippage = market_price - slippage
-
-    return Fill(fill_datetime, simple_order.quantity, fill_price_with_slippage)
+    return Fill(
+        fill_datetime, simple_order.quantity, market_price, includes_slippage=False
+    )

sysobjects/instruments.py

@@ -328,13 +328,20 @@ def calculate_cost_percentage_terms(
         return cost_in_percentage_terms
 
     def calculate_cost_instrument_currency(
-        self, blocks_traded: float, block_price_multiplier: float, price: float
+        self,
+        blocks_traded: float,
+        block_price_multiplier: float,
+        price: float,
+        include_slippage: bool = True,
     ) -> float:
 
         value_per_block = price * block_price_multiplier
-        slippage = self.calculate_slippage_instrument_currency(
-            blocks_traded, block_price_multiplier=block_price_multiplier
-        )
+        if include_slippage:
+            slippage = self.calculate_slippage_instrument_currency(
+                blocks_traded, block_price_multiplier=block_price_multiplier
+            )
+        else:
+            slippage = 0
 
         commission = self.calculate_total_commission(
             blocks_traded, value_per_block=value_per_block