From e2b60958b783ff84e4722d00f5b689e888fe7f3d Mon Sep 17 00:00:00 2001 From: todd <3578666+tgibson11@users.noreply.github.com> Date: Thu, 8 Jun 2023 05:46:36 -0700 Subject: [PATCH 1/5] Raise missingInstrument exception when unable to get FX config --- sysbrokers/IB/config/ib_fx_config.py | 5 +++-- sysbrokers/IB/ib_Fx_prices_data.py | 9 ++++----- syscore/constants.py | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/sysbrokers/IB/config/ib_fx_config.py b/sysbrokers/IB/config/ib_fx_config.py index dfc8ce3e07..798481067e 100644 --- a/sysbrokers/IB/config/ib_fx_config.py +++ b/sysbrokers/IB/config/ib_fx_config.py @@ -2,7 +2,8 @@ import pandas as pd -from syscore.constants import missing_file, missing_instrument +from syscore.constants import missing_file +from syscore.exceptions import missingInstrument from syscore.fileutils import resolve_path_and_filename_for_package from syslogging.logger import * @@ -31,7 +32,7 @@ def config_info_for_code(config_data: pd.DataFrame, currency_code, log) -> ibFXC "Can't get IB FX config for %s as config file missing" % currency_code ) - return missing_instrument + raise missingInstrument ccy1 = config_data[config_data.CODE == currency_code].CCY1.values[0] ccy2 = config_data[config_data.CODE == currency_code].CCY2.values[0] diff --git a/sysbrokers/IB/ib_Fx_prices_data.py b/sysbrokers/IB/ib_Fx_prices_data.py index 2b8860f8c9..306636f1fd 100644 --- a/sysbrokers/IB/ib_Fx_prices_data.py +++ b/sysbrokers/IB/ib_Fx_prices_data.py @@ -8,11 +8,10 @@ ibFXConfig, ) from sysbrokers.broker_fx_prices_data import brokerFxPricesData -from syscore.exceptions import missingData +from syscore.exceptions import missingData, missingInstrument from sysdata.data_blob import dataBlob from sysobjects.spot_fx_prices import fxPrices from syslogging.logger import * -from syscore.constants import missing_instrument class ibFxPricesData(brokerFxPricesData): @@ -45,9 +44,9 @@ def get_list_of_fxcodes(self) -> list: return list_of_codes def _get_fx_prices_without_checking(self, currency_code: str) -> fxPrices: - ib_config_for_code = self._get_config_info_for_code(currency_code) - - if ib_config_for_code is missing_instrument: + try: + ib_config_for_code = self._get_config_info_for_code(currency_code) + except missingInstrument: log = self.log.setup(**{CURRENCY_CODE_LOG_LABEL: currency_code}) log.warn("Can't get prices as missing IB config for %s" % currency_code) return fxPrices.create_empty() diff --git a/syscore/constants.py b/syscore/constants.py index a9e9e44343..f7afbdb406 100644 --- a/syscore/constants.py +++ b/syscore/constants.py @@ -9,7 +9,6 @@ def __repr__(self): return self._name -missing_instrument = named_object("missing instrument") missing_file = named_object("missing file") market_closed = named_object("market closed") fill_exceeds_trade = named_object("fill too big for trade") From bbc0882625f1dc6d295c5ab9c1d3f7e7d64a1e1a Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 9 Jun 2023 15:39:54 +0100 Subject: [PATCH 2/5] version 1.62 --- docs/backtesting.md | 2 +- examples/introduction/simplesystem.py | 2 +- syscore/genutils.py | 4 + sysdata/production/historic_orders.py | 10 +- sysobjects/fills.py | 6 +- sysobjects/orders.py | 46 +++- sysproduction/data/orders.py | 6 +- .../strategy_code/report_system_classic.py | 8 +- .../accounts/account_buffering_subsystem.py | 4 +- systems/accounts/account_costs.py | 10 +- .../accounts/account_curve_order_simulator.py | 170 ++++++++++++++ systems/accounts/account_inputs.py | 15 +- systems/accounts/account_instruments.py | 2 +- systems/accounts/account_subsystem.py | 11 +- .../pandl_order_simulator.py | 215 ++++++++++++++++++ .../pandl_calculators/pandl_using_fills.py | 18 +- systems/diagoutput.py | 2 +- systems/portfolio.py | 6 +- systems/positionsizing.py | 4 +- .../accounts_stage.py | 2 +- .../example/hourly_with_order_simulation.py | 52 +++++ .../example/hourly_with_order_simulator.yaml | 27 +++ systems/provided/mr/accounts.py | 36 +++ systems/provided/mr/config.yaml | 4 +- systems/provided/mr/create_orders.py | 202 ++++++++++++++++ .../provided/mr/mr_pandl_order_simulator.py | 129 +++++++++++ systems/provided/mr/rawdata.py | 11 + systems/provided/mr/rules.py | 22 +- .../optimise_small_system.py | 2 +- systems/tests/test_position_sizing.py | 2 +- tests/test_examples.py | 6 +- 31 files changed, 979 insertions(+), 57 deletions(-) create mode 100644 systems/accounts/account_curve_order_simulator.py create mode 100644 systems/accounts/pandl_calculators/pandl_order_simulator.py create mode 100644 systems/provided/example/hourly_with_order_simulation.py create mode 100644 systems/provided/example/hourly_with_order_simulator.yaml create mode 100644 systems/provided/mr/accounts.py create mode 100644 systems/provided/mr/create_orders.py create mode 100644 systems/provided/mr/mr_pandl_order_simulator.py create mode 100644 systems/provided/mr/rawdata.py diff --git a/docs/backtesting.md b/docs/backtesting.md index 0220bca6e2..bdc2db1f78 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -4397,7 +4397,7 @@ Other methods exist to access logging and caching. | `positionSize.get_block_value` | Standard | `instrument_code` | D | Get value of a 1% move in the price | | `positionSize.get_instrument_currency_vol` | Standard | `instrument_code` |D | Get daily volatility in the currency of the instrument | | `positionSize.get_instrument_value_vol` | Standard | `instrument_code` |D | Get daily volatility in the currency of the trading account | -| `positionSize.get_volatility_scalar` | Standard | `instrument_code` | D |Get ratio of target volatility vs volatility of instrument in instrument's own currency | +| `positionSize.get_average_position_at_subsystem_level` | Standard | `instrument_code` | D |Get ratio of target volatility vs volatility of instrument in instrument's own currency | | `positionSize.get_subsystem_position`| Standard | `instrument_code` | D, O |Get position if we put our entire trading capital into one instrument | diff --git a/examples/introduction/simplesystem.py b/examples/introduction/simplesystem.py index 23141be605..2990c2a560 100644 --- a/examples/introduction/simplesystem.py +++ b/examples/introduction/simplesystem.py @@ -150,7 +150,7 @@ print(my_system.positionSize.get_block_value("SOFR").tail(5)) print(my_system.positionSize.get_underlying_price("SOFR")) print(my_system.positionSize.get_instrument_value_vol("SOFR").tail(5)) -print(my_system.positionSize.get_volatility_scalar("SOFR").tail(5)) +print(my_system.positionSize.get_average_position_at_subsystem_level("SOFR").tail(5)) print(my_system.positionSize.get_vol_target_dict()) print(my_system.positionSize.get_subsystem_position("SOFR").tail(5)) diff --git a/syscore/genutils.py b/syscore/genutils.py index ea00d84f71..401373104a 100755 --- a/syscore/genutils.py +++ b/syscore/genutils.py @@ -132,6 +132,10 @@ def str_of_int(x: int) -> str: return "" +def same_sign(x, y): + return sign(x) == sign(y) + + def sign(x: Union[int, float]) -> float: """ >>> sign(3) diff --git a/sysdata/production/historic_orders.py b/sysdata/production/historic_orders.py index 2ec2a799b4..075371f6f3 100644 --- a/sysdata/production/historic_orders.py +++ b/sysdata/production/historic_orders.py @@ -19,7 +19,7 @@ from sysexecution.orders.named_order_objects import missing_order from sysdata.base_data import baseData -from sysobjects.fills import listOfFills, fill_from_order +from sysobjects.fills import ListOfFills, fill_from_order from sysexecution.orders.base_orders import Order from sysexecution.orders.broker_orders import single_fill_from_broker_order from sysexecution.order_stacks.order_stack import missingOrder @@ -71,7 +71,7 @@ def get_list_of_order_ids_in_date_range( class strategyHistoricOrdersData(genericOrdersData): def get_fills_history_for_instrument_strategy( self, instrument_strategy: instrumentStrategy - ) -> listOfFills: + ) -> ListOfFills: """ :param instrument_code: str @@ -82,7 +82,7 @@ def get_fills_history_for_instrument_strategy( instrument_strategy ) order_list_as_fills = [fill_from_order(order) for order in order_list] - list_of_fills = listOfFills(order_list_as_fills) + list_of_fills = ListOfFills(order_list_as_fills) return list_of_fills @@ -115,7 +115,7 @@ class contractHistoricOrdersData(genericOrdersData): class brokerHistoricOrdersData(contractHistoricOrdersData): def get_fills_history_for_contract( self, futures_contract: futuresContract - ) -> listOfFills: + ) -> ListOfFills: """ :param instrument_code: str @@ -133,7 +133,7 @@ def get_fills_history_for_contract( for orderid in list_of_order_ids ] list_of_fills = [fill for fill in list_of_fills if fill is not missing_order] - list_of_fills = listOfFills(list_of_fills) + list_of_fills = ListOfFills(list_of_fills) return list_of_fills diff --git a/sysobjects/fills.py b/sysobjects/fills.py index 1b99b6634b..9b0ab5c440 100644 --- a/sysobjects/fills.py +++ b/sysobjects/fills.py @@ -15,7 +15,7 @@ NOT_FILLED = named_object("not filled") -class listOfFills(list): +class ListOfFills(list): def __init__(self, list_of_fills): list_of_fills = [fill for fill in list_of_fills if fill is not missing_order] super().__init__(list_of_fills) @@ -47,7 +47,7 @@ def from_position_series_and_prices(cls, positions: pd.Series, price: pd.Series) def _list_of_fills_from_position_series_and_prices( positions: pd.Series, price: pd.Series -) -> listOfFills: +) -> ListOfFills: ( trades_without_zeros, @@ -63,7 +63,7 @@ def _list_of_fills_from_position_series_and_prices( for date, qty, price in zip(dates_as_list, trades_as_list, prices_as_list) ] - list_of_fills = listOfFills(list_of_fills_as_list) + list_of_fills = ListOfFills(list_of_fills_as_list) return list_of_fills diff --git a/sysobjects/orders.py b/sysobjects/orders.py index 5b23407ab8..f2f45a1836 100644 --- a/sysobjects/orders.py +++ b/sysobjects/orders.py @@ -1,5 +1,9 @@ +from collections import namedtuple +from typing import List +import datetime from dataclasses import dataclass -from syscore.constants import named_object, arg_not_supplied + +from syscore.pandas.pdutils import make_df_from_list_of_named_tuple @dataclass() @@ -28,6 +32,46 @@ def zero_order(cls): class ListOfSimpleOrders(list): + def __init__(self, list_of_orders: List[SimpleOrder]): + super().__init__(list_of_orders) + def remove_zero_orders(self): new_list = [order for order in self if not order.is_zero_order] return ListOfSimpleOrders(new_list) + + +_SimpleOrderWithDateAsTuple = namedtuple( + "_SimpleOrderWithDateAsTuple", ["submit_date", "quantity", "limit_price"] +) + + +class SimpleOrderWithDate(SimpleOrder): + def __init__( + self, quantity: int, submit_date: datetime.datetime, limit_price: float = None + ): + super().__init__(quantity=quantity, limit_price=limit_price) + self.submit_date = submit_date + + @classmethod + def zero_order(cls, submit_date: datetime.datetime): + return cls(quantity=0, submit_date=submit_date) + + def _as_tuple(self): + return _SimpleOrderWithDateAsTuple( + submit_date=self.submit_date, + quantity=self.quantity, + limit_price=self.limit_price, + ) + + +class ListOfSimpleOrdersWithDate(ListOfSimpleOrders): + def __init__(self, list_of_orders: List[SimpleOrderWithDate]): + super().__init__(list_of_orders) + + def as_pd_df(self): + return make_df_from_list_of_named_tuple( + _SimpleOrderWithDateAsTuple, self._as_list_of_named_tuples() + ) + + def _as_list_of_named_tuples(self) -> list: + return [order._as_tuple() for order in self] diff --git a/sysproduction/data/orders.py b/sysproduction/data/orders.py index 9ea2c63d0b..bc7caaea10 100644 --- a/sysproduction/data/orders.py +++ b/sysproduction/data/orders.py @@ -19,7 +19,7 @@ ) from sysdata.data_blob import dataBlob -from sysobjects.fills import listOfFills +from sysobjects.fills import ListOfFills from sysexecution.order_stacks.broker_order_stack import brokerOrderStackData from sysexecution.order_stacks.contract_order_stack import contractOrderStackData from sysexecution.order_stacks.instrument_order_stack import instrumentOrderStackData @@ -160,7 +160,7 @@ def get_historic_broker_order_from_order_id(self, order_id: int) -> brokerOrder: def get_fills_history_for_contract( self, futures_contract: futuresContract - ) -> listOfFills: + ) -> ListOfFills: ## We get this from broker fills, as they have leg by leg information list_of_fills = ( self.db_broker_historic_orders_data.get_fills_history_for_contract( @@ -172,7 +172,7 @@ def get_fills_history_for_contract( def get_fills_history_for_instrument_strategy( self, instrument_strategy: instrumentStrategy - ) -> listOfFills: + ) -> ListOfFills: list_of_fills = self.db_strategy_historic_orders_data.get_fills_history_for_instrument_strategy( instrument_strategy ) diff --git a/sysproduction/strategy_code/report_system_classic.py b/sysproduction/strategy_code/report_system_classic.py index 2a225bea09..ed6678ce81 100644 --- a/sysproduction/strategy_code/report_system_classic.py +++ b/sysproduction/strategy_code/report_system_classic.py @@ -304,7 +304,13 @@ def get_forecast_matrix_over_code( ) get_vol_scalar = configForMethod( - "positionSize", "get_volatility_scalar", "Vol Scalar", False, True, None, False + "positionSize", + "get_average_position_at_subsystem_level", + "Vol Scalar", + False, + True, + None, + False, ) get_subsystem_position = configForMethod( diff --git a/systems/accounts/account_buffering_subsystem.py b/systems/accounts/account_buffering_subsystem.py index a3e529022a..1556d6b57b 100644 --- a/systems/accounts/account_buffering_subsystem.py +++ b/systems/accounts/account_buffering_subsystem.py @@ -12,7 +12,9 @@ class accountBufferingSubSystemLevel(accountCosts): def subsystem_turnover(self, instrument_code: str) -> float: positions = self.get_subsystem_position(instrument_code) - average_position_for_turnover = self.get_volatility_scalar(instrument_code) + average_position_for_turnover = self.get_average_position_at_subsystem_level( + instrument_code + ) subsystem_turnover = turnover(positions, average_position_for_turnover) diff --git a/systems/accounts/account_costs.py b/systems/accounts/account_costs.py index eb72428dbd..bfb676c31e 100644 --- a/systems/accounts/account_costs.py +++ b/systems/accounts/account_costs.py @@ -344,7 +344,9 @@ def get_SR_cost_per_trade_for_instrument_percentage( @diagnostic() def _recent_average_price(self, instrument_code: str) -> float: - daily_price = self.get_instrument_prices_for_position_or_forecast(instrument_code) + daily_price = self.get_instrument_prices_for_position_or_forecast( + instrument_code + ) start_date = self._date_one_year_before_end_of_price_index(instrument_code) average_price = float(daily_price[start_date:].mean()) @@ -352,7 +354,9 @@ def _recent_average_price(self, instrument_code: str) -> float: @diagnostic() def _date_one_year_before_end_of_price_index(self, instrument_code: str): - daily_price = self.get_instrument_prices_for_position_or_forecast(instrument_code) + daily_price = self.get_instrument_prices_for_position_or_forecast( + instrument_code + ) last_date = daily_price.index[-1] start_date = last_date - pd.DateOffset(years=1) @@ -378,5 +382,5 @@ def _recent_average_daily_vol(self, instrument_code: str) -> float: return average_vol @property - def use_SR_costs(self) -> float: + def use_SR_costs(self) -> bool: return str2Bool(self.config.use_SR_costs) diff --git a/systems/accounts/account_curve_order_simulator.py b/systems/accounts/account_curve_order_simulator.py new file mode 100644 index 0000000000..17aa239bbd --- /dev/null +++ b/systems/accounts/account_curve_order_simulator.py @@ -0,0 +1,170 @@ +import pandas as pd + +from systems.system_cache import diagnostic + +from systems.accounts.pandl_calculators.pandl_cash_costs import ( + pandlCalculationWithCashCostsAndFills, +) + +from systems.accounts.curves.account_curve import accountCurve +from systems.accounts.accounts_stage import Account +from systems.accounts.pandl_calculators.pandl_order_simulator import ( + OrderSimulator, + HourlyOrderSimulatorOfMarketOrders, +) + + +class AccountWithOrderSimulator(Account): + @diagnostic(not_pickable=True) + def pandl_for_subsystem( + self, instrument_code, delayfill=True, roundpositions=True + ) -> accountCurve: + + self.log.msg( + "Calculating pandl for subsystem for instrument %s" % instrument_code, + instrument_code=instrument_code, + ) + + use_SR_costs = self.use_SR_costs + _raise_exceptions( + roundpositions=roundpositions, + delayfill=delayfill, + use_SR_costs=use_SR_costs, + ) + + order_simulator = self.get_order_simulator(instrument_code, is_subsystem=False) + price = order_simulator.prices() + fills = order_simulator.list_of_fills() + + raw_costs = self.get_raw_cost_data(instrument_code) + fx = self.get_fx_rate(instrument_code) + value_of_price_point = self.get_value_of_block_price_move(instrument_code) + capital = self.get_notional_capital() + vol_normalise_currency_costs = self.config.vol_normalise_currency_costs + rolls_per_year = self.get_rolls_per_year(instrument_code) + + pandl_calculator = pandlCalculationWithCashCostsAndFills( + price, + raw_costs=raw_costs, + fills=fills, + capital=capital, + value_per_point=value_of_price_point, + delayfill=delayfill, + fx=fx, + roundpositions=roundpositions, + vol_normalise_currency_costs=vol_normalise_currency_costs, + rolls_per_year=rolls_per_year, + ) + + account_curve = accountCurve(pandl_calculator) + + return account_curve + + @diagnostic(not_pickable=True) + def pandl_for_instrument( + self, instrument_code: str, delayfill: bool = True, roundpositions: bool = True + ) -> accountCurve: + self.log.msg( + "Calculating pandl for instrument for %s" % instrument_code, + instrument_code=instrument_code, + ) + use_SR_costs = self.use_SR_costs + _raise_exceptions( + roundpositions=roundpositions, + delayfill=delayfill, + use_SR_costs=use_SR_costs, + ) + + order_simulator = self.get_order_simulator(instrument_code, is_subsystem=False) + fills = order_simulator.list_of_fills() + price = order_simulator.prices() + + raw_costs = self.get_raw_cost_data(instrument_code) + + fx = self.get_fx_rate(instrument_code) + value_of_price_point = self.get_value_of_block_price_move(instrument_code) + + capital = self.get_notional_capital() + + vol_normalise_currency_costs = self.config.vol_normalise_currency_costs + rolls_per_year = self.get_rolls_per_year(instrument_code) + multiply_roll_costs_by = self.config.multiply_roll_costs_by + + pandl_calculator = pandlCalculationWithCashCostsAndFills( + price, + raw_costs=raw_costs, + fills=fills, + capital=capital, + value_per_point=value_of_price_point, + delayfill=delayfill, + fx=fx, + roundpositions=roundpositions, + vol_normalise_currency_costs=vol_normalise_currency_costs, + rolls_per_year=rolls_per_year, + multiply_roll_costs_by=multiply_roll_costs_by, + ) + + account_curve = accountCurve(pandl_calculator, weighted=True) + + return account_curve + + @diagnostic() + def get_buffered_position( + self, instrument_code: str, roundpositions: bool = True + ) -> pd.Series: + _raise_exceptions(roundpositions=roundpositions) + order_simulator = self.get_order_simulator(instrument_code, is_subsystem=False) + + return order_simulator.positions() + + @diagnostic() + def get_buffered_subsystem_position( + self, instrument_code: str, roundpositions: bool = True + ) -> pd.Series: + _raise_exceptions(roundpositions=roundpositions) + order_simulator = self.get_order_simulator(instrument_code, is_subsystem=True) + + return order_simulator.positions() + + @diagnostic(not_pickable=True) + def get_order_simulator( + self, instrument_code, is_subsystem: bool + ) -> OrderSimulator: + raise NotImplemented("Need to inherit to get an order simulator") + + +## example +class AccountWithOrderSimulatorForHourlyMarketOrders(AccountWithOrderSimulator): + @diagnostic(not_pickable=True) + def get_order_simulator( + self, instrument_code, is_subsystem: bool + ) -> HourlyOrderSimulatorOfMarketOrders: + order_simulator = HourlyOrderSimulatorOfMarketOrders( + system_accounts_stage=self, + instrument_code=instrument_code, + is_subsystem=is_subsystem, + ) + return order_simulator + + def get_unrounded_subsystem_position_for_order_simulator( + self, instrument_code: str + ) -> pd.Series: + return self.get_subsystem_position(instrument_code) + + def get_unrounded_instrument_position_for_order_simulator( + self, instrument_code: str + ) -> pd.Series: + return self.get_notional_position(instrument_code) + + +def _raise_exceptions( + roundpositions: bool = True, use_SR_costs: bool = False, delayfill: bool = True +): + if not roundpositions: + raise Exception("Have to round positions when using order simulator!") + if not delayfill: + raise Exception("Have to delay fills when using order simulator!") + if use_SR_costs: + raise Exception( + "Have to use cash costs not SR costs when using order simulator!" + ) diff --git a/systems/accounts/account_inputs.py b/systems/accounts/account_inputs.py index 3f39397851..24ef92d7fa 100644 --- a/systems/accounts/account_inputs.py +++ b/systems/accounts/account_inputs.py @@ -98,6 +98,9 @@ def target_abs_forecast(self) -> float: def average_forecast(self) -> float: return self.config.average_absolute_forecast + def forecast_cap(self) -> float: + return self.config.forecast_cap + def get_raw_cost_data(self, instrument_code: str) -> instrumentCosts: return self.parent.data.get_raw_cost_data(instrument_code) @@ -151,7 +154,9 @@ def get_annual_risk_target(self) -> float: def get_average_position_for_instrument_at_portfolio_level( self, instrument_code: str ) -> pd.Series: - average_position_for_subsystem = self.get_volatility_scalar(instrument_code) + average_position_for_subsystem = self.get_average_position_at_subsystem_level( + instrument_code + ) scaling_factor = self.get_instrument_scaling_factor(instrument_code) scaling_factor_aligned = scaling_factor.reindex( average_position_for_subsystem.index, method="ffill" @@ -160,7 +165,9 @@ def get_average_position_for_instrument_at_portfolio_level( return average_position - def get_volatility_scalar(self, instrument_code: str) -> pd.Series: + def get_average_position_at_subsystem_level( + self, instrument_code: str + ) -> pd.Series: """ Get the volatility scalar (position with forecast of +10 using all capital) @@ -173,7 +180,9 @@ def get_volatility_scalar(self, instrument_code: str) -> pd.Series: """ - return self.parent.positionSize.get_volatility_scalar(instrument_code) + return self.parent.positionSize.get_average_position_at_subsystem_level( + instrument_code + ) def get_notional_position(self, instrument_code: str) -> pd.Series: """ diff --git a/systems/accounts/account_instruments.py b/systems/accounts/account_instruments.py index bdea58756a..65643aac2f 100644 --- a/systems/accounts/account_instruments.py +++ b/systems/accounts/account_instruments.py @@ -162,7 +162,7 @@ def turnover_at_portfolio_level( ) -> float: ## assumes we use all capital - average_position_for_turnover = self.get_volatility_scalar(instrument_code) + average_position_for_turnover = self.get_average_position_at_subsystem_level(instrument_code) ## Using actual capital positions = self.get_buffered_position( diff --git a/systems/accounts/account_subsystem.py b/systems/accounts/account_subsystem.py index 0bad50ca3f..69ed67b2fc 100644 --- a/systems/accounts/account_subsystem.py +++ b/systems/accounts/account_subsystem.py @@ -106,7 +106,9 @@ def _pandl_for_subsystem_with_SR_costs( ) -> accountCurve: positions = self.get_buffered_subsystem_position(instrument_code) - price = self.get_instrument_prices_for_position_or_forecast(instrument_code, position_or_forecast=positions) + price = self.get_instrument_prices_for_position_or_forecast( + instrument_code, position_or_forecast=positions + ) fx = self.get_fx_rate(instrument_code) @@ -114,7 +116,7 @@ def _pandl_for_subsystem_with_SR_costs( daily_returns_volatility = self.get_daily_returns_volatility(instrument_code) ## following doesn't include IDM or instrument weight - average_position = self.get_volatility_scalar(instrument_code) + average_position = self.get_average_position_at_subsystem_level(instrument_code) subsystem_turnover = self.subsystem_turnover(instrument_code) annualised_SR_cost = self.get_SR_cost_given_turnover( @@ -147,8 +149,9 @@ def _pandl_for_subsystem_with_cash_costs( raw_costs = self.get_raw_cost_data(instrument_code) positions = self.get_buffered_subsystem_position(instrument_code) - price = self.get_instrument_prices_for_position_or_forecast(instrument_code, - position_or_forecast=positions) ### here! + price = self.get_instrument_prices_for_position_or_forecast( + instrument_code, position_or_forecast=positions + ) ### here! fx = self.get_fx_rate(instrument_code) diff --git a/systems/accounts/pandl_calculators/pandl_order_simulator.py b/systems/accounts/pandl_calculators/pandl_order_simulator.py new file mode 100644 index 0000000000..baf37ef014 --- /dev/null +++ b/systems/accounts/pandl_calculators/pandl_order_simulator.py @@ -0,0 +1,215 @@ +import numpy as np +import datetime +from typing import Tuple +from dataclasses import dataclass +import pandas as pd +from syscore.cache import Cache + +from sysobjects.orders import SimpleOrderWithDate, ListOfSimpleOrdersWithDate +from sysobjects.fills import ListOfFills, Fill + + +@dataclass +class PositionsOrdersFills: + positions: pd.Series + list_of_orders: ListOfSimpleOrdersWithDate + list_of_fills: ListOfFills + + +@dataclass +class OrderSimulator: + system_accounts_stage: object ## no explicit type as would cause circular import + instrument_code: str + is_subsystem: bool = False + + def diagnostic_df(self) -> pd.DataFrame: + return self.cache.get(self._diagnostic_df) + + def _diagnostic_df(self) -> pd.DataFrame: + raise NotImplemented("Need to inherit from this class to get diagnostics") + + def prices(self) -> pd.Series: + raise NotImplemented("Need to inherit from this class to get prices") + + def positions(self) -> pd.Series: + positions_orders_fills = self.positions_orders_and_fills_from_series_data() + return positions_orders_fills.positions + + def list_of_fills(self) -> ListOfFills: + positions_orders_fills = self.positions_orders_and_fills_from_series_data() + return positions_orders_fills.list_of_fills + + def list_of_orders(self) -> ListOfSimpleOrdersWithDate: + positions_orders_fills = self.positions_orders_and_fills_from_series_data() + return positions_orders_fills.list_of_orders + + def positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: + ## Because p&l with orders is path dependent, we generate everything together + return self.cache.get(self._positions_orders_and_fills_from_series_data) + + def _positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: + raise NotImplemented( + "Need to inherit from this class and implement positions, orders, fills" + ) + + @property + def cache(self) -> Cache: + return getattr(self, "_cache", Cache(self)) + + +## Example to show how to do this + + +@dataclass +class HourlyMarketOrdersSeriesData: + hourly_price_series: pd.Series + hourly_unrounded_positions: pd.Series + + +class HourlyOrderSimulatorOfMarketOrders(OrderSimulator): + def _diagnostic_df(self) -> pd.DataFrame: + position_series = self.positions() + position_df = pd.DataFrame(position_series) + + optimal_positions_series = self.optimal_positions_series() + optimal_position_df = pd.DataFrame(optimal_positions_series) + + list_of_fills = self.list_of_fills() + fills_df = list_of_fills.as_pd_df() + list_of_orders = self.list_of_orders() + orders_df = list_of_orders.as_pd_df() + df = pd.concat([optimal_position_df, orders_df, fills_df, position_df], axis=1) + df.columns = [ + "optimal_position", + "order_qty", + "limit_price", + "fill_qty", + "fill_price", + "position", + ] + + return df + + def prices(self) -> pd.Series: + return self.series_data.hourly_price_series + + def optimal_positions_series(self) -> pd.Series: + return self.series_data.hourly_unrounded_positions + + def _positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: + + positions_orders_fills = _generate_positions_orders_and_fills_from_hourly_series_data_for_market_orders( + self.series_data + ) + + return positions_orders_fills + + @property + def series_data(self) -> HourlyMarketOrdersSeriesData: + return self.cache.get(self._series_data) + + def _series_data(self) -> HourlyMarketOrdersSeriesData: + series_data = _build_series_data_for_order_simulator( + system_accounts_stage=self.system_accounts_stage, + instrument_code=self.instrument_code, + is_subsystem=self.is_subsystem, + ) + return series_data + + @property + def cache(self) -> Cache: + return getattr(self, "_cache", Cache(self)) + + +def _build_series_data_for_order_simulator( + system_accounts_stage, ## no explicit type would cause circular import + instrument_code: str, + is_subsystem: bool = False, +) -> HourlyMarketOrdersSeriesData: + + hourly_price_series = system_accounts_stage.get_hourly_prices(instrument_code) + if is_subsystem: + hourly_unrounded_positions = ( + system_accounts_stage.get_unrounded_subsystem_position_for_order_simulator( + instrument_code + ) + ) + else: + hourly_unrounded_positions = ( + system_accounts_stage.get_unrounded_instrument_position_for_order_simulator( + instrument_code + ) + ) + + series_data = HourlyMarketOrdersSeriesData( + hourly_price_series=hourly_price_series, + hourly_unrounded_positions=hourly_unrounded_positions, + ) + return series_data + + +def _generate_positions_orders_and_fills_from_hourly_series_data_for_market_orders( + series_data: HourlyMarketOrdersSeriesData, +) -> PositionsOrdersFills: + + unrounded_positions = series_data.hourly_unrounded_positions + hourly_prices = series_data.hourly_price_series + + list_of_positions = [] + list_of_orders = [] + list_of_fills = [] + + starting_position = 0 ## doesn't do anything but makes intention clear + current_position = starting_position + + for idx, current_datetime in enumerate(unrounded_positions.index[:-1]): + list_of_positions.append(current_position) + + current_optimal_position = unrounded_positions[idx] + next_price = hourly_prices[idx + 1] + next_datetime = unrounded_positions.index[idx + 1] + + order, fill = _generate_order_and_fill_at_idx_point( + current_position=current_position, + current_optimal_position=current_optimal_position, + current_datetime=current_datetime, + next_price=next_price, + next_datetime=next_datetime, + ) + if not order.is_zero_order: + list_of_orders.append(order) + list_of_fills.append(fill) + current_position = current_position + fill.qty + + ## Because we don't loop at the final point as no fill is possible, we keep our last position + ## This ensures the list of positions has the same index as the unrounded list + list_of_positions.append(current_position) + + positions = pd.Series(list_of_positions, unrounded_positions.index) + list_of_orders = ListOfSimpleOrdersWithDate(list_of_orders) + list_of_fills = ListOfFills(list_of_fills) + + return PositionsOrdersFills( + positions=positions, list_of_orders=list_of_orders, list_of_fills=list_of_fills + ) + + +def _generate_order_and_fill_at_idx_point( + current_position: int, + current_optimal_position: int, + current_datetime: datetime.datetime, + next_datetime: datetime.datetime, + next_price: float, +) -> Tuple[SimpleOrderWithDate, Fill]: + if np.isnan(current_optimal_position): + quantity = 0 + else: + quantity = round(current_optimal_position) - current_position + + simple_order = SimpleOrderWithDate( + quantity=quantity, + submit_date=current_datetime, + ) + fill = Fill(date=next_datetime, price=next_price, qty=simple_order.quantity) + + return simple_order, fill diff --git a/systems/accounts/pandl_calculators/pandl_using_fills.py b/systems/accounts/pandl_calculators/pandl_using_fills.py index 95ecb38b7a..a4a1b8c24b 100644 --- a/systems/accounts/pandl_calculators/pandl_using_fills.py +++ b/systems/accounts/pandl_calculators/pandl_using_fills.py @@ -7,11 +7,11 @@ apply_weighting, ) -from sysobjects.fills import listOfFills, Fill +from sysobjects.fills import ListOfFills, Fill class pandlCalculationWithFills(pandlCalculation): - def __init__(self, *args, fills: listOfFills = arg_not_supplied, **kwargs): + def __init__(self, *args, fills: ListOfFills = arg_not_supplied, **kwargs): # if fills aren't supplied, can be inferred from positions super().__init__(*args, **kwargs) self._fills = fills @@ -38,7 +38,7 @@ def using_positions_and_prices_merged_from_fills( pandlCalculation, price: pd.Series, positions: pd.Series, - fills: listOfFills, + fills: ListOfFills, **kwargs, ): @@ -47,7 +47,7 @@ def using_positions_and_prices_merged_from_fills( return pandlCalculation(price=merged_prices, positions=positions, **kwargs) @property - def fills(self) -> listOfFills: + def fills(self) -> ListOfFills: fills = self._fills if fills is arg_not_supplied: # Infer from positions @@ -59,14 +59,14 @@ def fills(self) -> listOfFills: return fills - def _infer_fills_from_position(self) -> listOfFills: + def _infer_fills_from_position(self) -> ListOfFills: # positions will have delayfill and round applied to them already positions = self.positions if positions is arg_not_supplied: raise Exception("Need to pass fills or positions") - fills = listOfFills.from_position_series_and_prices( + fills = ListOfFills.from_position_series_and_prices( positions=positions, price=self.price ) return fills @@ -104,7 +104,7 @@ def _calculate_and_set_prices_from_fills_and_input_prices(self) -> pd.Series: def merge_fill_prices_with_prices( - prices: pd.Series, list_of_fills: listOfFills + prices: pd.Series, list_of_fills: ListOfFills ) -> pd.Series: list_of_trades_as_pd_df = list_of_fills.as_pd_df() unique_trades_as_pd_df = unique_trades_df(list_of_trades_as_pd_df) @@ -137,7 +137,7 @@ def unique_trades_df(trade_df: pd.DataFrame) -> pd.DataFrame: return new_df -def infer_positions_from_fills(fills: listOfFills) -> pd.Series: +def infer_positions_from_fills(fills: ListOfFills) -> pd.Series: date_index = [fill.date for fill in fills] qty_trade = [fill.qty for fill in fills] trade_series = pd.Series(qty_trade, index=date_index) @@ -153,7 +153,7 @@ def infer_positions_from_fills(fills: listOfFills) -> pd.Series: @classmethod def using_fills(pandlCalculation, price: pd.Series, - fills: listOfFills, + fills: ListOfFills, **kwargs): positions = from_fills_to_positions(fills) diff --git a/systems/diagoutput.py b/systems/diagoutput.py index eb3c75bbd5..5e726dce66 100644 --- a/systems/diagoutput.py +++ b/systems/diagoutput.py @@ -299,7 +299,7 @@ def calculation_details(self, instrument_code): "positionSize.get_instrument_currency_vol", "positionSize.get_fx_rate", "positionSize.get_instrument_value_vol", - "positionSize.get_volatility_scalar", + "positionSize.get_average_position_at_subsystem_level", "positionSize.get_subsystem_position", "portfolio.get_notional_position", ] diff --git a/systems/portfolio.py b/systems/portfolio.py index 27ca08a444..42a4d56e47 100644 --- a/systems/portfolio.py +++ b/systems/portfolio.py @@ -257,8 +257,8 @@ def get_notional_position_without_idm(self, instrument_code: str) -> pd.Series: instrument_weight_this_code = instr_weights[instrument_code] inst_weight_this_code_reindexed = instrument_weight_this_code.reindex( - subsys_position.index - ).ffill() + subsys_position.index, method="ffill" + ) notional_position_without_idm = ( subsys_position * inst_weight_this_code_reindexed @@ -915,7 +915,7 @@ def get_volatility_scalar(self, instrument_code: str) -> pd.Series: ()], data, config) >>> >>> ## from config - >>> system.portfolio.get_volatility_scalar("EDOLLAR").tail(2) + >>> system.portfolio.get_average_position_at_subsystem_level("EDOLLAR").tail(2) vol_scalar 2015-12-10 11.187869 2015-12-11 10.332930 diff --git a/systems/positionsizing.py b/systems/positionsizing.py index 599c3280a0..cdcff21d4a 100644 --- a/systems/positionsizing.py +++ b/systems/positionsizing.py @@ -178,14 +178,14 @@ def get_volatility_scalar(self, instrument_code: str) -> pd.Series: >>> (comb, fcs, rules, rawdata, data, config)=get_test_object_futures_with_comb_forecasts() >>> system=System([rawdata, rules, fcs, comb, PositionSizing()], data, config) >>> - >>> system.positionSize.get_volatility_scalar("EDOLLAR").tail(2) + >>> system.positionSize.get_average_position_at_subsystem_level("EDOLLAR").tail(2) vol_scalar 2015-12-10 11.187869 2015-12-11 10.332930 >>> >>> ## without raw data >>> system2=System([ rules, fcs, comb, PositionSizing()], data, config) - >>> system2.positionSize.get_volatility_scalar("EDOLLAR").tail(2) + >>> system2.positionSize.get_average_position_at_subsystem_level("EDOLLAR").tail(2) vol_scalar 2015-12-10 11.180444 2015-12-11 10.344278 diff --git a/systems/provided/dynamic_small_system_optimise/accounts_stage.py b/systems/provided/dynamic_small_system_optimise/accounts_stage.py index fdde5408e9..335c8fcff6 100644 --- a/systems/provided/dynamic_small_system_optimise/accounts_stage.py +++ b/systems/provided/dynamic_small_system_optimise/accounts_stage.py @@ -71,7 +71,7 @@ def optimised_turnover_at_portfolio_level( ) -> float: ## assumes we use all capital - average_position_for_turnover = self.get_volatility_scalar(instrument_code) + average_position_for_turnover = self.get_average_position_at_subsystem_level(instrument_code) ## Using actual capital positions = self.get_optimised_position(instrument_code) diff --git a/systems/provided/example/hourly_with_order_simulation.py b/systems/provided/example/hourly_with_order_simulation.py new file mode 100644 index 0000000000..d1cd2a00fc --- /dev/null +++ b/systems/provided/example/hourly_with_order_simulation.py @@ -0,0 +1,52 @@ +### THIS IS AN EXAMPLE OF HOW TO USE A PROPER ORDER SIMULATOR RATHER THAN VECTORISED +### P&L, FOR A SIMPLE TREND SYSTEM USING HOURLY DATA WITH MARKET ORDERS + +import matplotlib + +matplotlib.use("TkAgg") + +from syscore.constants import arg_not_supplied + +# from sysdata.sim.csv_futures_sim_data import csvFuturesSimData +from sysdata.sim.db_futures_sim_data import dbFuturesSimData +from sysdata.config.configdata import Config + +from systems.forecasting import Rules +from systems.basesystem import System + +from systems.rawdata import RawData +from systems.forecast_combine import ForecastCombine +from systems.forecast_scale_cap import ForecastScaleCap +from systems.positionsizing import PositionSizing +from systems.portfolio import Portfolios +from systems.accounts.account_curve_order_simulator import ( + AccountWithOrderSimulatorForHourlyMarketOrders, +) + + +def futures_system( + sim_data=arg_not_supplied, + config_filename="systems.provided.example.hourly_with_order_simulator.yaml", +): + + if sim_data is arg_not_supplied: + sim_data = dbFuturesSimData() + + config = Config(config_filename) + + system = System( + [ + AccountWithOrderSimulatorForHourlyMarketOrders(), + Portfolios(), + PositionSizing(), + ForecastCombine(), + ForecastScaleCap(), + Rules(), + RawData(), + ], + sim_data, + config, + ) + system.set_logging_level("on") + + return system diff --git a/systems/provided/example/hourly_with_order_simulator.yaml b/systems/provided/example/hourly_with_order_simulator.yaml new file mode 100644 index 0000000000..0e9f2bcfa1 --- /dev/null +++ b/systems/provided/example/hourly_with_order_simulator.yaml @@ -0,0 +1,27 @@ +#YAML +percentage_vol_target: 25 +notional_trading_capital: 50000 +base_currency: "USD" +trading_rules: + ewmac8: + function: systems.provided.rules.ewmac.ewmac_forecast_with_defaults + data: rawdata.get_hourly_prices + other_args: + Lfast: 8 + Lslow: 32 + forecast_scalar: 20.0 + ewmac32: + function: systems.provided.rules.ewmac.ewmac_forecast_with_defaults + data: rawdata.get_hourly_prices + other_args: + Lfast: 32 + Lslow: 128 + forecast_scalar: 10.0 +forecast_weights: + ewmac8: 0.50 + ewmac32: 0.50 +forecast_div_multiplier: 1.1 +instrument_weights: + US10: .5 + SP500_micro: .5 +instrument_div_multiplier: 1.4 diff --git a/systems/provided/mr/accounts.py b/systems/provided/mr/accounts.py new file mode 100644 index 0000000000..1f63bd0c3f --- /dev/null +++ b/systems/provided/mr/accounts.py @@ -0,0 +1,36 @@ +import pandas as pd + +from systems.system_cache import dont_cache, diagnostic, input +from systems.accounts.curves.account_curve import accountCurve +from systems.accounts.account_curve_order_simulator import AccountWithOrderSimulator + +from systems.provided.mr.forecast_combine import MrForecastCombine +from systems.provided.mr.rawdata import MrRawData +from systems.provided.mr.mr_pandl_order_simulator import MROrderSimulator + + +class MrAccount(AccountWithOrderSimulator): + @diagnostic(not_pickable=True) + def get_order_simulator(self, instrument_code, subsystem: bool) -> MROrderSimulator: + return MROrderSimulator( + system_accounts_stage=self, + instrument_code=instrument_code, + subsystem=subsystem, + ) + + ### The following is required to access information to do the MR positions + @input + def daily_equilibrium_price(self, instrument_code: str) -> pd.Series: + return self.raw_data_stage.daily_equilibrium_price(instrument_code) + + @input + def conditioning_forecast(self, instrument_code: str) -> pd.Series: + return self.comb_forecast_stage.conditioning_forecast(instrument_code) + + @property + def comb_forecast_stage(self) -> MrForecastCombine: + return self.parent.combForecast + + @property + def raw_data_stage(self) -> MrRawData: + return self.parent.rawdata diff --git a/systems/provided/mr/config.yaml b/systems/provided/mr/config.yaml index d37b590e09..a3a7478689 100644 --- a/systems/provided/mr/config.yaml +++ b/systems/provided/mr/config.yaml @@ -1,4 +1,5 @@ mr: + mr_span_days: 5 conditioning_rule: momentum16 mr_rule: mean_reversion ## this rule is used for the @@ -17,8 +18,7 @@ trading_rules: - "rawdata.get_hourly_prices" - "rawdata.get_daily_prices" - "rawdata.daily_returns_volatility" - other_args: - mr_span_days: 5 + - "rawdata.daily_equilibrium_price" forecast_scalars: momentum16: 4.1 mean_reversion: 20.0 diff --git a/systems/provided/mr/create_orders.py b/systems/provided/mr/create_orders.py new file mode 100644 index 0000000000..35c6b70bc9 --- /dev/null +++ b/systems/provided/mr/create_orders.py @@ -0,0 +1,202 @@ +from typing import Union +from dataclasses import dataclass +from syscore.genutils import same_sign +from syscore.constants import named_object, arg_not_supplied +from sysobjects.orders import SimpleOrder + +CLOSE_MR_POSITIONS = named_object("close mr positions") +REACHED_FORECAST_LIMIT = named_object("reached forecast limit") + + +@dataclass +class DataForMROrder: + current_position: int + current_equilibrium_price: float + current_price: float + current_vol: float + current_conditioner_for_forecast: float + average_position: float + avg_abs_forecast: float = 10.0 + mr_forecast_scalar: float = 20.0 + lower_forecast_floor: float = -20.0 + upper_forecast_cap: float = 20.0 + + def limit_price_given_higher_position( + self, position_to_derive_for + ) -> Union[float, named_object]: + return self._derive_limit_price_at_position_for_mean_reversion_overlay( + position_to_derive_for, upper_position=True + ) + + def limit_price_given_lower_position( + self, position_to_derive_for + ) -> Union[float, named_object]: + return self._derive_limit_price_at_position_for_mean_reversion_overlay( + position_to_derive_for, upper_position=False + ) + + def _derive_limit_price_at_position_for_mean_reversion_overlay( + self, position_to_derive_for: int, upper_position: bool = True + ) -> Union[float, named_object]: + """ + + If forecast = cap, then don't return a price if too high + If forecast turned off by conditioning, then fail + + """ + + scaled_forecast = self.scaled_forecast + + if not same_sign(scaled_forecast, self.current_conditioner_for_forecast): + ## Market order + return CLOSE_MR_POSITIONS + + limit_price = self._derive_limit_price_at_position_for_mean_reversion_overlay_when_conditioning_on( + position_to_derive_for=position_to_derive_for, upper_position=upper_position + ) + + return limit_price + + def _derive_limit_price_at_position_for_mean_reversion_overlay_when_conditioning_on( + self, position_to_derive_for: int, upper_position: bool = True + ) -> Union[float, named_object]: + + if self._is_forecast_beyond_limits(upper_position): + return REACHED_FORECAST_LIMIT + + limit_price = self.derive_limit_price_without_checks(position_to_derive_for) + + return limit_price + + def _is_forecast_beyond_limits(self, upper_position: bool) -> bool: + + lower_position = not upper_position + if self.scaled_forecast > self.upper_forecast_cap and upper_position: + return True + elif self.scaled_forecast < self.lower_forecast_floor and lower_position: + return True + else: + return False + + def derive_limit_price_without_checks(self, position_to_derive_for: int) -> float: + """ + FORECAST = FORECAST_SCALAR * (equilibrium - current_price)/ daily_vol_hourly + POSITION = (average position * forecast ) / (AVG_ABS_FORECAST) + = (average position * FORECAST_SCALAR * (equilibrium - current_price) / (AVG_ABS_FORECAST * daily_vol_hourly) + + POSITION * (AVG_ABS_FORECAST * daily_vol_hourly) = average position * FORECAST_SCALAR * (equilibrium - current_price) + (equilibrium - current_price) = POSITION * AVG_ABS_FORECAST * daily_vol_hourly / (average position * FORECAST SCALAR) + current price = equilibrium - [POSITION * AVG_ABS_FORECAST * daily_vol_hourly / (average position * FORECAST SCALAR)] + """ + + limit_price = self.current_equilibrium_price - ( + position_to_derive_for * self.avg_abs_forecast * self.current_vol + ) / (self.average_position * self.mr_forecast_scalar) + + return limit_price + + @property + def capped_scaled_forecast(self) -> float: + capped_scaled_forecast = min( + [ + max([self.scaled_forecast, self.lower_forecast_floor]), + self.upper_forecast_cap, + ] + ) + + return capped_scaled_forecast + + @property + def scaled_forecast(self) -> float: + raw_forecast = ( + self.current_equilibrium_price - self.current_price + ) / self.current_vol + + scaled_forecast = raw_forecast * self.mr_forecast_scalar + + return scaled_forecast + + +def create_orders_from_mr_data(data_for_mr_order: DataForMROrder) -> list: + + optimal_position = derive_unrounded_position(data_for_mr_order) + if optimal_position is CLOSE_MR_POSITIONS: + ## market order to close positions + return [SimpleOrder(-data_for_mr_order.current_position)] + list_of_orders = _create_orders_for_mr_data_if_not_closing( + optimal_position=optimal_position, data_for_mr_order=data_for_mr_order + ) + + return list_of_orders + + +def derive_unrounded_position( + data_for_mr_order: DataForMROrder, +) -> Union[float, named_object]: + + capped_scaled_forecast = data_for_mr_order.capped_scaled_forecast + if not same_sign( + capped_scaled_forecast, data_for_mr_order.current_conditioner_for_forecast + ): + ## Market order + return CLOSE_MR_POSITIONS + + position = ( + data_for_mr_order.average_position + * capped_scaled_forecast + / data_for_mr_order.avg_abs_forecast + ) + + return position + + +def _create_orders_for_mr_data_if_not_closing( + optimal_position: float, data_for_mr_order: DataForMROrder +) -> list: + rounded_optimal = round(optimal_position) + diff_to_current = abs(rounded_optimal - data_for_mr_order.current_position) + + if diff_to_current > 1: + ## close everything no limit order + return [SimpleOrder(rounded_optimal - data_for_mr_order.current_position)] + + list_of_orders = _create_limit_orders_for_mr_data(data_for_mr_order) + + return list_of_orders + + +def _create_limit_orders_for_mr_data(data_for_mr_order: DataForMROrder) -> list: + lower_order_list = _create_lower_limit_order(data_for_mr_order) + upper_order_list = _create_upper_limit_order(data_for_mr_order) + list_of_orders = lower_order_list + upper_order_list + + return list_of_orders + + +def _create_upper_limit_order(data_for_mr_order: DataForMROrder) -> list: + trade_to_upper = +1 + position_upper = data_for_mr_order.current_position + trade_to_upper + upper_limit_price = data_for_mr_order.limit_price_given_higher_position( + position_upper + ) + if upper_limit_price is REACHED_FORECAST_LIMIT: + upper_order_list = [] + else: + upper_order_list = [SimpleOrder(trade_to_upper, upper_limit_price)] + + return upper_order_list + + +def _create_lower_limit_order(data_for_mr_order: DataForMROrder) -> list: + trade_to_lower = -1 + position_lower = data_for_mr_order.current_position + trade_to_lower + lower_limit_price = data_for_mr_order.limit_price_given_lower_position( + position_lower + ) + + if lower_limit_price is REACHED_FORECAST_LIMIT: + lower_order_list = [] + else: + lower_order_list = [SimpleOrder(trade_to_lower, lower_limit_price)] + + return lower_order_list diff --git a/systems/provided/mr/mr_pandl_order_simulator.py b/systems/provided/mr/mr_pandl_order_simulator.py new file mode 100644 index 0000000000..8008faeee1 --- /dev/null +++ b/systems/provided/mr/mr_pandl_order_simulator.py @@ -0,0 +1,129 @@ +from typing import Tuple +from dataclasses import dataclass +import pandas as pd + + +from syscore.cache import Cache +from systems.provided.mr.create_orders import DataForMROrder, create_orders_from_mr_data +from systems.provided.mr.accounts import MrAccount + +from systems.accounts.curves.account_curve import accountCurve +from systems.accounts.pandl_calculators.pandl_order_simulator import OrderSimulator +from systems.accounts.accounts_stage import Account +from sysobjects.orders import SimpleOrder, ListOfSimpleOrders + + +@dataclass +class MROrderSeriesData: + equilibrium_hourly_price_series: pd.Series + hourly_price_series: pd.Series + hourly_vol_series: pd.Series + conditioning_forecast_series: pd.Series + average_position_series: pd.Series + avg_abs_forecast: float = 10.0 + abs_forecast_cap: float = 20.0 + + +class MROrderSimulator(OrderSimulator): + system_accounts_stage: MrAccount + instrument_code: str + subsystem: bool = False + + def _positions_orders_and_fills_from_series_data(self): + mr_order_series_data = build_mr_series_data( + self.system_accounts_stage, + instrument_code=self.instrument_code, + is_subsystem=self.subsystem, + ) + return generate_positions_orders_and_fills_from_series_data( + mr_order_series_data=mr_order_series_data + ) + + +def build_mr_series_data( + system_accounts_stage: MrAccount, + instrument_code: str, + is_subsystem: bool = False, +) -> MROrderSeriesData: + + daily_equilibrium = system_accounts_stage.daily_equilibrium_price(instrument_code) + equilibrium_hourly_price_series = daily_equilibrium.reindex() + hourly_price_series = system_accounts_stage.get_hourly_prices(instrument_code) + daily_vol_series = system_accounts_stage.get_daily_returns_volatility( + instrument_code + ) + hourly_vol_series = daily_vol_series.reindex( + hourly_price_series.index, method="ffill" + ) + daily_conditioning_forecast_series = system_accounts_stage.conditioning_forecast( + instrument_code + ) + conditioning_forecast_series = daily_conditioning_forecast_series.reindex( + hourly_price_series.index, method="ffiil" + ) + if is_subsystem: + daily_average_position_series = ( + system_accounts_stage.get_average_position_at_subsystem_level( + instrument_code + ) + ) + else: + daily_average_position_series = system_accounts_stage.get_average_position_for_instrument_at_portfolio_level( + instrument_code + ) + + average_position_series = daily_average_position_series.reindex( + hourly_price_series.index, method="ffill" + ) + avg_abs_forecast = system_accounts_stage.average_forecast() + abs_forecast_cap = system_accounts_stage.forecast_cap() + + return MROrderSeriesData( + equilibrium_hourly_price_series=equilibrium_hourly_price_series, + average_position_series=average_position_series, + hourly_price_series=hourly_price_series, + hourly_vol_series=hourly_vol_series, + conditioning_forecast_series=conditioning_forecast_series, + avg_abs_forecast=avg_abs_forecast, + abs_forecast_cap=abs_forecast_cap, + ) + + +def generate_positions_orders_and_fills_from_series_data( + mr_order_series_data: MROrderSeriesData, + delayfill: bool = True, +) -> Tuple[ListOfSimpleOrders, list, list]: + + current_position = 0 + list_of_orders = [] + list_of_fills = [] + list_of_positions = [] + master_date_index = mr_order_series_data.hourly_price_series.index + + for idx in range(len(master_date_index)): + ## requires mr_data to be index matched + ( + new_position, + orders, + fill, + ) = _generate_positions_orders_and_fills_from_series_data_for_idx( + idx=idx, + current_position=current_position, + mr_order_series_data=mr_order_series_data, + delayfill=delayfill, + ) + list_of_fills.append(fill) + list_of_orders.append(orders) + list_of_positions.append(new_position) + + ## list of orders and fills need to be timestamped + return list_of_positions, list_of_orders, list_of_fills + + +def _generate_positions_orders_and_fills_from_series_data_for_idx( + idx: int, + current_position: int, + mr_order_series_data: MROrderSeriesData, + delayfill: bool = True, +) -> Tuple[int, order, fill]: + pass diff --git a/systems/provided/mr/rawdata.py b/systems/provided/mr/rawdata.py new file mode 100644 index 0000000000..0153511665 --- /dev/null +++ b/systems/provided/mr/rawdata.py @@ -0,0 +1,11 @@ +import pandas as pd +from systems.rawdata import RawData + + +class MrRawData(RawData): + def daily_equilibrium_price(self, instrument_code: str) -> pd.Series: + daily_price = self.get_daily_prices(instrument_code) + mr_span = self.config.mr["mr_span_days"] + daily_equilibrium = daily_price.ewm(span=mr_span).mean() + + return daily_equilibrium diff --git a/systems/provided/mr/rules.py b/systems/provided/mr/rules.py index f21c8ff5db..c3bda12858 100644 --- a/systems/provided/mr/rules.py +++ b/systems/provided/mr/rules.py @@ -1,12 +1,16 @@ import pandas as pd -def mr_rule(hourly_price: pd.Series, - daily_price: pd.Series, - daily_vol: pd.Series, - mr_span_days: int = 5) -> pd.Series: - equilibrium = daily_price.ewm(span=mr_span_days).mean() - daily_vol_hourly = daily_vol.reindex(hourly_price.index, method = "ffill") - equilibrium = equilibrium.reindex(hourly_price.index, method = "ffill") - forecast_before_filter = (equilibrium - hourly_price)/ daily_vol_hourly - return forecast_before_filter \ No newline at end of file +def mr_rule( + hourly_price: pd.Series, + daily_vol: pd.Series, + daily_equilibrium: pd.Series, +) -> pd.Series: + daily_vol_indexed_hourly = daily_vol.reindex(hourly_price.index, method="ffill") + hourly_equilibrium = daily_equilibrium.reindex(hourly_price.index, method="ffill") + + forecast_before_filter = ( + hourly_equilibrium - hourly_price + ) / daily_vol_indexed_hourly + + return forecast_before_filter diff --git a/systems/provided/static_small_system_optimise/optimise_small_system.py b/systems/provided/static_small_system_optimise/optimise_small_system.py index de71e539ae..b34b658a84 100644 --- a/systems/provided/static_small_system_optimise/optimise_small_system.py +++ b/systems/provided/static_small_system_optimise/optimise_small_system.py @@ -281,7 +281,7 @@ def net_SR_for_instrument_in_system( def calculate_maximum_position(system, instrument_code, instrument_weight_idm=0.25): - pos_at_average = system.positionSize.get_volatility_scalar(instrument_code) + pos_at_average = system.positionSize.get_average_position_at_subsystem_level(instrument_code) pos_at_average_in_system = pos_at_average * instrument_weight_idm forecast_multiplier = ( system.combForecast.get_forecast_cap() / system.positionSize.avg_abs_forecast() diff --git a/systems/tests/test_position_sizing.py b/systems/tests/test_position_sizing.py index 6e2c6712d9..b0676dc816 100644 --- a/systems/tests/test_position_sizing.py +++ b/systems/tests/test_position_sizing.py @@ -123,7 +123,7 @@ def test_get_instrument_value_vol(self): @unittest.SkipTest def test_get_get_volatility_scalar(self): self.assertAlmostEqual( - self.system.positionSize.get_volatility_scalar("EDOLLAR") + self.system.positionSize.get_average_position_at_subsystem_level("EDOLLAR") .ffill() .values[-1], 10.33292952955, diff --git a/tests/test_examples.py b/tests/test_examples.py index d05a6ed96c..e5f159315e 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -222,7 +222,11 @@ def test_simple_system_position_sizing( print(my_system.positionSize.get_block_value("EDOLLAR").tail(5)) print(my_system.positionSize.get_underlying_price("EDOLLAR")) print(my_system.positionSize.get_instrument_value_vol("EDOLLAR").tail(5)) - print(my_system.positionSize.get_volatility_scalar("EDOLLAR").tail(5)) + print( + my_system.positionSize.get_average_position_at_subsystem_level( + "EDOLLAR" + ).tail(5) + ) print(my_system.positionSize.get_vol_target_dict()) print(my_system.positionSize.get_subsystem_position("EDOLLAR").tail(5)) From f0da8a0fc6980a1f2d3f2a19fd283fded6ca7439 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 9 Jun 2023 15:40:48 +0100 Subject: [PATCH 3/5] version 1.62 --- CHANGELOG.md | 7 +++++++ README.md | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf01bf389c..a4d4b9a670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Release notes +## Version 1.62 + +- Added order simulator as an optimal replacement for vectorised p&l calculation; prequisite for limit order simulation +- Replace pst logging with python logging +- ignore daily expiries for certain EUREX contracts +- Allow fixed instrument and forecast weights to be specificed as a hierarchy + ## Version 1.61 - Replaced log to database with log to file diff --git a/README.md b/README.md index 96d5d0f074..16b608ed3b 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ Rob Carver [https://qoppac.blogspot.com/p/pysystemtrade.html](https://qoppac.blogspot.com/p/pysystemtrade.html) -Version 1.61 +Version 1.62 -2023-03-24 +2023-06-09 From 6a28dcb2ee69298b1bb1fe41f29b162efdb0d616 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 9 Jun 2023 15:41:59 +0100 Subject: [PATCH 4/5] black is back --- systems/accounts/account_forecast.py | 5 +-- systems/accounts/account_instruments.py | 12 +++++-- .../accounts_stage.py | 4 ++- systems/provided/mr/forecast_combine.py | 36 ++++++++++++------- systems/provided/mr/run_system.py | 2 ++ .../provided/rob_system/forecastScaleCap.py | 4 ++- .../optimise_small_system.py | 4 ++- 7 files changed, 47 insertions(+), 20 deletions(-) diff --git a/systems/accounts/account_forecast.py b/systems/accounts/account_forecast.py index 427a841858..66b37c5654 100644 --- a/systems/accounts/account_forecast.py +++ b/systems/accounts/account_forecast.py @@ -186,8 +186,9 @@ def pandl_for_instrument_forecast( forecast = self.get_capped_forecast(instrument_code, rule_variation_name) - price = self.get_instrument_prices_for_position_or_forecast(instrument_code=instrument_code, - position_or_forecast=forecast) + price = self.get_instrument_prices_for_position_or_forecast( + instrument_code=instrument_code, position_or_forecast=forecast + ) daily_returns_volatility = self.get_daily_returns_volatility(instrument_code) diff --git a/systems/accounts/account_instruments.py b/systems/accounts/account_instruments.py index 65643aac2f..4f0d1080ad 100644 --- a/systems/accounts/account_instruments.py +++ b/systems/accounts/account_instruments.py @@ -121,7 +121,9 @@ def _pandl_for_instrument_with_SR_costs( roundpositions: bool = True, ) -> accountCurve: - price = self.get_instrument_prices_for_position_or_forecast(instrument_code, position_or_forecast=positions) + price = self.get_instrument_prices_for_position_or_forecast( + instrument_code, position_or_forecast=positions + ) fx = self.get_fx_rate(instrument_code) value_of_price_point = self.get_value_of_block_price_move(instrument_code) daily_returns_volatility = self.get_daily_returns_volatility(instrument_code) @@ -162,7 +164,9 @@ def turnover_at_portfolio_level( ) -> float: ## assumes we use all capital - average_position_for_turnover = self.get_average_position_at_subsystem_level(instrument_code) + average_position_for_turnover = self.get_average_position_at_subsystem_level( + instrument_code + ) ## Using actual capital positions = self.get_buffered_position( @@ -188,7 +192,9 @@ def _pandl_for_instrument_with_cash_costs( raw_costs = self.get_raw_cost_data(instrument_code) - price = self.get_instrument_prices_for_position_or_forecast(instrument_code, position_or_forecast=positions) + price = self.get_instrument_prices_for_position_or_forecast( + instrument_code, position_or_forecast=positions + ) fx = self.get_fx_rate(instrument_code) value_of_price_point = self.get_value_of_block_price_move(instrument_code) diff --git a/systems/provided/dynamic_small_system_optimise/accounts_stage.py b/systems/provided/dynamic_small_system_optimise/accounts_stage.py index 335c8fcff6..9e71c0fff5 100644 --- a/systems/provided/dynamic_small_system_optimise/accounts_stage.py +++ b/systems/provided/dynamic_small_system_optimise/accounts_stage.py @@ -71,7 +71,9 @@ def optimised_turnover_at_portfolio_level( ) -> float: ## assumes we use all capital - average_position_for_turnover = self.get_average_position_at_subsystem_level(instrument_code) + average_position_for_turnover = self.get_average_position_at_subsystem_level( + instrument_code + ) ## Using actual capital positions = self.get_optimised_position(instrument_code) diff --git a/systems/provided/mr/forecast_combine.py b/systems/provided/mr/forecast_combine.py index 01bc4aa589..ded6b9761a 100644 --- a/systems/provided/mr/forecast_combine.py +++ b/systems/provided/mr/forecast_combine.py @@ -5,6 +5,7 @@ from systems.system_cache import output, diagnostic from systems.forecast_combine import ForecastCombine + class MrForecastCombine(ForecastCombine): @output() def get_combined_forecast(self, instrument_code: str) -> pd.Series: @@ -14,36 +15,47 @@ def get_combined_forecast(self, instrument_code: str) -> pd.Series: ## this is already scaled and capped forecast_before_filter = self.mr_forecast(instrument_code) - forecast_after_filter = apply_forecast_filter(forecast_before_filter, conditioning_forecast) + forecast_after_filter = apply_forecast_filter( + forecast_before_filter, conditioning_forecast + ) forecast_after_filter = forecast_after_filter.ffill() return forecast_after_filter - def conditioning_forecast(self, instrument_code) -> pd.Series: - conditioning_rule_name = self.config.mr['conditioning_rule'] - return self._get_capped_individual_forecast(instrument_code=instrument_code, - rule_variation_name=conditioning_rule_name) + conditioning_rule_name = self.config.mr["conditioning_rule"] + return self._get_capped_individual_forecast( + instrument_code=instrument_code, rule_variation_name=conditioning_rule_name + ) def mr_forecast(self, instrument_code) -> pd.Series: - mr_rule_name = self.config.mr['mr_rule'] - return self._get_capped_individual_forecast(instrument_code=instrument_code, - rule_variation_name=mr_rule_name) + mr_rule_name = self.config.mr["mr_rule"] + return self._get_capped_individual_forecast( + instrument_code=instrument_code, rule_variation_name=mr_rule_name + ) + def apply_forecast_filter(forecast, conditioning_forecast): - conditioning_forecast = conditioning_forecast.reindex(forecast.index, method = "ffill") + conditioning_forecast = conditioning_forecast.reindex( + forecast.index, method="ffill" + ) - new_values = [forecast_overlay_for_sign(forecast_value, filter_value) - for forecast_value, filter_value - in zip(forecast.values, conditioning_forecast.values)] + new_values = [ + forecast_overlay_for_sign(forecast_value, filter_value) + for forecast_value, filter_value in zip( + forecast.values, conditioning_forecast.values + ) + ] return pd.Series(new_values, forecast.index) + def forecast_overlay_for_sign(forecast_value, filter_value): if same_sign(forecast_value, filter_value): return forecast_value else: return 0 + def same_sign(x, y): return sign(x) == sign(y) diff --git a/systems/provided/mr/run_system.py b/systems/provided/mr/run_system.py index 5c0578578a..9048f173c8 100644 --- a/systems/provided/mr/run_system.py +++ b/systems/provided/mr/run_system.py @@ -1,4 +1,5 @@ import matplotlib + matplotlib.use("TkAgg") from syscore.constants import arg_not_supplied @@ -16,6 +17,7 @@ from systems.portfolio import Portfolios from systems.accounts.accounts_stage import Account + def futures_system( sim_data=arg_not_supplied, config_filename="systems.provided.mr.config.yaml" ): diff --git a/systems/provided/rob_system/forecastScaleCap.py b/systems/provided/rob_system/forecastScaleCap.py index 0b29e15935..6dab8ee128 100644 --- a/systems/provided/rob_system/forecastScaleCap.py +++ b/systems/provided/rob_system/forecastScaleCap.py @@ -45,7 +45,9 @@ def get_raw_forecast(self, instrument_code, rule_variation_name): return raw_forecast_before_atten else: vol_attenutation = self.get_vol_attenuation(instrument_code) - vol_attenutation_reindex = vol_attenutation.reindex(raw_forecast_before_atten.index, method="ffill") + vol_attenutation_reindex = vol_attenutation.reindex( + raw_forecast_before_atten.index, method="ffill" + ) attenuated_forecast = raw_forecast_before_atten * vol_attenutation_reindex return attenuated_forecast diff --git a/systems/provided/static_small_system_optimise/optimise_small_system.py b/systems/provided/static_small_system_optimise/optimise_small_system.py index b34b658a84..c22fff0043 100644 --- a/systems/provided/static_small_system_optimise/optimise_small_system.py +++ b/systems/provided/static_small_system_optimise/optimise_small_system.py @@ -281,7 +281,9 @@ def net_SR_for_instrument_in_system( def calculate_maximum_position(system, instrument_code, instrument_weight_idm=0.25): - pos_at_average = system.positionSize.get_average_position_at_subsystem_level(instrument_code) + pos_at_average = system.positionSize.get_average_position_at_subsystem_level( + instrument_code + ) pos_at_average_in_system = pos_at_average * instrument_weight_idm forecast_multiplier = ( system.combForecast.get_forecast_cap() / system.positionSize.avg_abs_forecast() From 39215240be5cc8a39a1bb55c3d6d71fcb390564f Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 9 Jun 2023 17:59:00 +0100 Subject: [PATCH 5/5] added limit orders to order simulator --- sysexecution/orders/named_order_objects.py | 1 + sysobjects/fills.py | 86 +++++++++--- sysobjects/instruments.py | 15 ++- systems/accounts/order_simulator/__init__.py | 0 .../account_curve_order_simulator.py | 29 +--- .../order_simulator/hourly_limit_orders.py | 124 ++++++++++++++++++ .../hourly_market_orders.py} | 94 +++++-------- .../order_simulator/pandl_order_simulator.py | 54 ++++++++ .../pandl_calculators/pandl_cash_costs.py | 2 + .../example/hourly_with_order_simulation.py | 13 +- systems/provided/mr/accounts.py | 7 +- .../provided/mr/mr_pandl_order_simulator.py | 9 +- 12 files changed, 310 insertions(+), 124 deletions(-) create mode 100644 systems/accounts/order_simulator/__init__.py rename systems/accounts/{ => order_simulator}/account_curve_order_simulator.py (83%) create mode 100644 systems/accounts/order_simulator/hourly_limit_orders.py rename systems/accounts/{pandl_calculators/pandl_order_simulator.py => order_simulator/hourly_market_orders.py} (73%) create mode 100644 systems/accounts/order_simulator/pandl_order_simulator.py diff --git a/sysexecution/orders/named_order_objects.py b/sysexecution/orders/named_order_objects.py index 942ac0e96f..e83068aa4d 100644 --- a/sysexecution/orders/named_order_objects.py +++ b/sysexecution/orders/named_order_objects.py @@ -1,5 +1,6 @@ from syscore.constants import named_object +not_filled = named_object("not filled") missing_order = named_object("missing order") locked_order = named_object("locked order") duplicate_order = named_object("duplicate order") diff --git a/sysobjects/fills.py b/sysobjects/fills.py index 9b0ab5c440..190b875b6a 100644 --- a/sysobjects/fills.py +++ b/sysobjects/fills.py @@ -1,23 +1,35 @@ +from typing import Union import datetime +from dataclasses import dataclass from collections import namedtuple import pandas as pd from syscore.constants import named_object -from sysexecution.orders.named_order_objects import missing_order -from sysobjects.orders import SimpleOrder, ListOfSimpleOrders +from sysexecution.orders.named_order_objects import missing_order, not_filled +from sysobjects.orders import ( + SimpleOrder, + ListOfSimpleOrders, + ListOfSimpleOrdersWithDate, + SimpleOrderWithDate, +) -from sysexecution.orders.list_of_orders import listOfOrders from sysexecution.orders.base_orders import Order -Fill = namedtuple("Fill", ["date", "qty", "price"]) -NOT_FILLED = named_object("not filled") +@dataclass +class Fill: + date: datetime.datetime + qty: int + price: float + includes_slippage: bool = False class ListOfFills(list): def __init__(self, list_of_fills): - list_of_fills = [fill for fill in list_of_fills if fill is not missing_order] + list_of_fills = [ + fill for fill in list_of_fills if fill is not (missing_order or not_filled) + ] super().__init__(list_of_fills) def _as_dict_of_lists(self) -> dict: @@ -103,20 +115,44 @@ def fill_from_order(order: Order) -> Fill: return Fill(fill_datetime, fill_qty, fill_price) +def fill_list_of_simple_orders( + list_of_orders: ListOfSimpleOrders, + fill_datetime: datetime.datetime, + market_price: float, +) -> Fill: + list_of_fills = [ + fill_from_simple_order( + simple_order=simple_order, + fill_datetime=fill_datetime, + market_price=market_price, + ) + for simple_order in list_of_orders + ] + list_of_fills = ListOfFills(list_of_fills) ## will remove unfilled + + if len(list_of_fills) == 0: + return not_filled + elif len(list_of_fills) == 1: + return list_of_fills[0] + else: + raise Exception( + "List of orders %s has produced more than one fill %s!" + % (str(list_of_orders), str(list_of_orders)) + ) + + def fill_from_simple_order( simple_order: SimpleOrder, market_price: float, fill_datetime: datetime.datetime, - slippage: float = 0, ) -> Fill: if simple_order.is_zero_order: - return NOT_FILLED + return not_filled elif simple_order.is_market_order: fill = fill_from_simple_market_order( simple_order, market_price=market_price, - slippage=slippage, fill_datetime=fill_datetime, ) else: @@ -129,31 +165,39 @@ def fill_from_simple_order( def fill_from_simple_limit_order( - simple_order: SimpleOrder, market_price: float, fill_datetime: datetime.datetime + simple_order: Union[SimpleOrder, SimpleOrderWithDate], + market_price: float, + fill_datetime: datetime.datetime, ) -> Fill: limit_price = simple_order.limit_price if simple_order.quantity > 0: if limit_price > market_price: - return Fill(fill_datetime, simple_order.quantity, limit_price) + return Fill( + fill_datetime, + simple_order.quantity, + limit_price, + includes_slippage=True, + ) if simple_order.quantity < 0: if limit_price < market_price: - return Fill(fill_datetime, simple_order.quantity, limit_price) + return Fill( + fill_datetime, + simple_order.quantity, + limit_price, + includes_slippage=True, + ) - return NOT_FILLED + return not_filled def fill_from_simple_market_order( - simple_order: SimpleOrder, + simple_order: Union[SimpleOrder, SimpleOrderWithDate], market_price: float, fill_datetime: datetime.datetime, - slippage: float = 0, ) -> Fill: - if simple_order.quantity > 0: - fill_price_with_slippage = market_price + slippage - else: - fill_price_with_slippage = market_price - slippage - - return Fill(fill_datetime, simple_order.quantity, fill_price_with_slippage) + return Fill( + fill_datetime, simple_order.quantity, market_price, includes_slippage=False + ) diff --git a/sysobjects/instruments.py b/sysobjects/instruments.py index 89735334a4..3d2c226140 100644 --- a/sysobjects/instruments.py +++ b/sysobjects/instruments.py @@ -328,13 +328,20 @@ def calculate_cost_percentage_terms( return cost_in_percentage_terms def calculate_cost_instrument_currency( - self, blocks_traded: float, block_price_multiplier: float, price: float + self, + blocks_traded: float, + block_price_multiplier: float, + price: float, + include_slippage: bool = True, ) -> float: value_per_block = price * block_price_multiplier - slippage = self.calculate_slippage_instrument_currency( - blocks_traded, block_price_multiplier=block_price_multiplier - ) + if include_slippage: + slippage = self.calculate_slippage_instrument_currency( + blocks_traded, block_price_multiplier=block_price_multiplier + ) + else: + slippage = 0 commission = self.calculate_total_commission( blocks_traded, value_per_block=value_per_block diff --git a/systems/accounts/order_simulator/__init__.py b/systems/accounts/order_simulator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/systems/accounts/account_curve_order_simulator.py b/systems/accounts/order_simulator/account_curve_order_simulator.py similarity index 83% rename from systems/accounts/account_curve_order_simulator.py rename to systems/accounts/order_simulator/account_curve_order_simulator.py index 17aa239bbd..27d9431b35 100644 --- a/systems/accounts/account_curve_order_simulator.py +++ b/systems/accounts/order_simulator/account_curve_order_simulator.py @@ -8,10 +8,7 @@ from systems.accounts.curves.account_curve import accountCurve from systems.accounts.accounts_stage import Account -from systems.accounts.pandl_calculators.pandl_order_simulator import ( - OrderSimulator, - HourlyOrderSimulatorOfMarketOrders, -) +from systems.accounts.order_simulator.pandl_order_simulator import OrderSimulator class AccountWithOrderSimulator(Account): @@ -133,30 +130,6 @@ def get_order_simulator( raise NotImplemented("Need to inherit to get an order simulator") -## example -class AccountWithOrderSimulatorForHourlyMarketOrders(AccountWithOrderSimulator): - @diagnostic(not_pickable=True) - def get_order_simulator( - self, instrument_code, is_subsystem: bool - ) -> HourlyOrderSimulatorOfMarketOrders: - order_simulator = HourlyOrderSimulatorOfMarketOrders( - system_accounts_stage=self, - instrument_code=instrument_code, - is_subsystem=is_subsystem, - ) - return order_simulator - - def get_unrounded_subsystem_position_for_order_simulator( - self, instrument_code: str - ) -> pd.Series: - return self.get_subsystem_position(instrument_code) - - def get_unrounded_instrument_position_for_order_simulator( - self, instrument_code: str - ) -> pd.Series: - return self.get_notional_position(instrument_code) - - def _raise_exceptions( roundpositions: bool = True, use_SR_costs: bool = False, delayfill: bool = True ): diff --git a/systems/accounts/order_simulator/hourly_limit_orders.py b/systems/accounts/order_simulator/hourly_limit_orders.py new file mode 100644 index 0000000000..7c10e02984 --- /dev/null +++ b/systems/accounts/order_simulator/hourly_limit_orders.py @@ -0,0 +1,124 @@ +import datetime +from typing import Tuple + +import numpy as np +import pandas as pd + +from sysobjects.fills import ListOfFills, Fill, fill_list_of_simple_orders, not_filled +from sysobjects.orders import ListOfSimpleOrdersWithDate, SimpleOrderWithDate + +from systems.accounts.order_simulator.pandl_order_simulator import ( + PositionsOrdersFills, +) +from systems.accounts.order_simulator.hourly_market_orders import ( + HourlyMarketOrdersSeriesData, + HourlyOrderSimulatorOfMarketOrders, + AccountWithOrderSimulatorForHourlyMarketOrders, +) +from systems.system_cache import diagnostic + + +class HourlyOrderSimulatorOfLimitOrders(HourlyOrderSimulatorOfMarketOrders): + def _positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: + + positions_orders_fills = _generate_positions_orders_and_fills_from_hourly_series_data_using_limit_orders( + self.series_data + ) + + return positions_orders_fills + + +## We only use limit orders +def _generate_positions_orders_and_fills_from_hourly_series_data_using_limit_orders( + series_data: HourlyMarketOrdersSeriesData, +) -> PositionsOrdersFills: + + unrounded_positions = series_data.hourly_unrounded_positions + hourly_prices = series_data.hourly_price_series + + list_of_positions = [] + list_of_orders = [] + list_of_fills = [] + + starting_position = 0 ## doesn't do anything but makes intention clear + current_position = starting_position + + for idx, current_datetime in enumerate(unrounded_positions.index[:-1]): + list_of_positions.append(current_position) + + current_optimal_position = unrounded_positions[idx] + current_price = hourly_prices[idx] + next_price = hourly_prices[idx + 1] + next_datetime = unrounded_positions.index[idx + 1] + + order, fill = _generate_limit_order_and_fill_at_idx_point( + current_position=current_position, + current_optimal_position=current_optimal_position, + current_datetime=current_datetime, + current_price=current_price, + next_price=next_price, + next_datetime=next_datetime, + ) + if not order.is_zero_order: + list_of_orders.append(order) + + filled_okay = not (fill is not_filled) + if filled_okay: + list_of_fills.append(fill) + current_position = current_position + fill.qty + + ## Because we don't loop at the final point as no fill is possible, we keep our last position + ## This ensures the list of positions has the same index as the unrounded list + list_of_positions.append(current_position) + + positions = pd.Series(list_of_positions, unrounded_positions.index) + list_of_orders = ListOfSimpleOrdersWithDate(list_of_orders) + list_of_fills = ListOfFills(list_of_fills) + + return PositionsOrdersFills( + positions=positions, list_of_orders=list_of_orders, list_of_fills=list_of_fills + ) + + +def _generate_limit_order_and_fill_at_idx_point( + current_position: int, + current_optimal_position: int, + current_datetime: datetime.datetime, + current_price: float, + next_datetime: datetime.datetime, + next_price: float, +) -> Tuple[SimpleOrderWithDate, Fill]: + if np.isnan(current_optimal_position): + quantity = 0 + else: + quantity = round(current_optimal_position) - current_position + + simple_order = SimpleOrderWithDate( + quantity=quantity, submit_date=current_datetime, limit_price=current_price + ) + simple_order_as_list = ListOfSimpleOrdersWithDate( + [simple_order] + ) ## future proofing for when we have 2 possible orders that can be filled + + fill = fill_list_of_simple_orders( + simple_order_as_list, + market_price=next_price, + fill_datetime=next_datetime, + ) + + return simple_order, fill + + +class AccountWithOrderSimulatorForLimitOrders( + AccountWithOrderSimulatorForHourlyMarketOrders +): + @diagnostic(not_pickable=True) + def get_order_simulator( + self, instrument_code, is_subsystem: bool + ) -> HourlyOrderSimulatorOfLimitOrders: + order_simulator = HourlyOrderSimulatorOfLimitOrders( + system_accounts_stage=self, + instrument_code=instrument_code, + is_subsystem=is_subsystem, + ) + return order_simulator diff --git a/systems/accounts/pandl_calculators/pandl_order_simulator.py b/systems/accounts/order_simulator/hourly_market_orders.py similarity index 73% rename from systems/accounts/pandl_calculators/pandl_order_simulator.py rename to systems/accounts/order_simulator/hourly_market_orders.py index baf37ef014..b2f994c626 100644 --- a/systems/accounts/pandl_calculators/pandl_order_simulator.py +++ b/systems/accounts/order_simulator/hourly_market_orders.py @@ -1,63 +1,22 @@ -import numpy as np import datetime -from typing import Tuple from dataclasses import dataclass +from typing import Tuple + +import numpy as np import pandas as pd -from syscore.cache import Cache -from sysobjects.orders import SimpleOrderWithDate, ListOfSimpleOrdersWithDate +from syscore.cache import Cache from sysobjects.fills import ListOfFills, Fill +from sysobjects.orders import ListOfSimpleOrdersWithDate, SimpleOrderWithDate - -@dataclass -class PositionsOrdersFills: - positions: pd.Series - list_of_orders: ListOfSimpleOrdersWithDate - list_of_fills: ListOfFills - - -@dataclass -class OrderSimulator: - system_accounts_stage: object ## no explicit type as would cause circular import - instrument_code: str - is_subsystem: bool = False - - def diagnostic_df(self) -> pd.DataFrame: - return self.cache.get(self._diagnostic_df) - - def _diagnostic_df(self) -> pd.DataFrame: - raise NotImplemented("Need to inherit from this class to get diagnostics") - - def prices(self) -> pd.Series: - raise NotImplemented("Need to inherit from this class to get prices") - - def positions(self) -> pd.Series: - positions_orders_fills = self.positions_orders_and_fills_from_series_data() - return positions_orders_fills.positions - - def list_of_fills(self) -> ListOfFills: - positions_orders_fills = self.positions_orders_and_fills_from_series_data() - return positions_orders_fills.list_of_fills - - def list_of_orders(self) -> ListOfSimpleOrdersWithDate: - positions_orders_fills = self.positions_orders_and_fills_from_series_data() - return positions_orders_fills.list_of_orders - - def positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: - ## Because p&l with orders is path dependent, we generate everything together - return self.cache.get(self._positions_orders_and_fills_from_series_data) - - def _positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: - raise NotImplemented( - "Need to inherit from this class and implement positions, orders, fills" - ) - - @property - def cache(self) -> Cache: - return getattr(self, "_cache", Cache(self)) - - -## Example to show how to do this +from systems.accounts.order_simulator.account_curve_order_simulator import ( + AccountWithOrderSimulator, +) +from systems.accounts.order_simulator.pandl_order_simulator import ( + OrderSimulator, + PositionsOrdersFills, +) +from systems.system_cache import diagnostic @dataclass @@ -116,10 +75,6 @@ def _series_data(self) -> HourlyMarketOrdersSeriesData: ) return series_data - @property - def cache(self) -> Cache: - return getattr(self, "_cache", Cache(self)) - def _build_series_data_for_order_simulator( system_accounts_stage, ## no explicit type would cause circular import @@ -213,3 +168,26 @@ def _generate_order_and_fill_at_idx_point( fill = Fill(date=next_datetime, price=next_price, qty=simple_order.quantity) return simple_order, fill + + +class AccountWithOrderSimulatorForHourlyMarketOrders(AccountWithOrderSimulator): + @diagnostic(not_pickable=True) + def get_order_simulator( + self, instrument_code, is_subsystem: bool + ) -> HourlyOrderSimulatorOfMarketOrders: + order_simulator = HourlyOrderSimulatorOfMarketOrders( + system_accounts_stage=self, + instrument_code=instrument_code, + is_subsystem=is_subsystem, + ) + return order_simulator + + def get_unrounded_subsystem_position_for_order_simulator( + self, instrument_code: str + ) -> pd.Series: + return self.get_subsystem_position(instrument_code) + + def get_unrounded_instrument_position_for_order_simulator( + self, instrument_code: str + ) -> pd.Series: + return self.get_notional_position(instrument_code) diff --git a/systems/accounts/order_simulator/pandl_order_simulator.py b/systems/accounts/order_simulator/pandl_order_simulator.py new file mode 100644 index 0000000000..71af9d2e7d --- /dev/null +++ b/systems/accounts/order_simulator/pandl_order_simulator.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass +import pandas as pd +from syscore.cache import Cache + +from sysobjects.orders import ListOfSimpleOrdersWithDate +from sysobjects.fills import ListOfFills + + +@dataclass +class PositionsOrdersFills: + positions: pd.Series + list_of_orders: ListOfSimpleOrdersWithDate + list_of_fills: ListOfFills + + +@dataclass +class OrderSimulator: + system_accounts_stage: object ## no explicit type as would cause circular import + instrument_code: str + is_subsystem: bool = False + + def diagnostic_df(self) -> pd.DataFrame: + return self.cache.get(self._diagnostic_df) + + def _diagnostic_df(self) -> pd.DataFrame: + raise NotImplemented("Need to inherit from this class to get diagnostics") + + def prices(self) -> pd.Series: + raise NotImplemented("Need to inherit from this class to get prices") + + def positions(self) -> pd.Series: + positions_orders_fills = self.positions_orders_and_fills_from_series_data() + return positions_orders_fills.positions + + def list_of_fills(self) -> ListOfFills: + positions_orders_fills = self.positions_orders_and_fills_from_series_data() + return positions_orders_fills.list_of_fills + + def list_of_orders(self) -> ListOfSimpleOrdersWithDate: + positions_orders_fills = self.positions_orders_and_fills_from_series_data() + return positions_orders_fills.list_of_orders + + def positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: + ## Because p&l with orders is path dependent, we generate everything together + return self.cache.get(self._positions_orders_and_fills_from_series_data) + + def _positions_orders_and_fills_from_series_data(self) -> PositionsOrdersFills: + raise NotImplemented( + "Need to inherit from this class and implement positions, orders, fills" + ) + + @property + def cache(self) -> Cache: + return getattr(self, "_cache", Cache(self)) diff --git a/systems/accounts/pandl_calculators/pandl_cash_costs.py b/systems/accounts/pandl_calculators/pandl_cash_costs.py index 3f23958b41..83901cc3d1 100644 --- a/systems/accounts/pandl_calculators/pandl_cash_costs.py +++ b/systems/accounts/pandl_calculators/pandl_cash_costs.py @@ -191,12 +191,14 @@ def normalise_costs_in_instrument_currency(self, costs_as_pd_series) -> pd.Serie def calculate_cost_instrument_currency_for_a_fill(self, fill: Fill) -> float: trade = fill.qty price = fill.price + include_slippage = fill.includes_slippage block_price_multiplier = self.value_per_point cost_for_trade = self.raw_costs.calculate_cost_instrument_currency( blocks_traded=trade, price=price, block_price_multiplier=block_price_multiplier, + include_slippage=include_slippage, ) return cost_for_trade diff --git a/systems/provided/example/hourly_with_order_simulation.py b/systems/provided/example/hourly_with_order_simulation.py index d1cd2a00fc..f80b60ba3e 100644 --- a/systems/provided/example/hourly_with_order_simulation.py +++ b/systems/provided/example/hourly_with_order_simulation.py @@ -19,13 +19,17 @@ from systems.forecast_scale_cap import ForecastScaleCap from systems.positionsizing import PositionSizing from systems.portfolio import Portfolios -from systems.accounts.account_curve_order_simulator import ( +from systems.accounts.order_simulator.hourly_market_orders import ( AccountWithOrderSimulatorForHourlyMarketOrders, ) +from systems.accounts.order_simulator.hourly_limit_orders import ( + AccountWithOrderSimulatorForLimitOrders, +) def futures_system( sim_data=arg_not_supplied, + use_limit_orders: bool = False, config_filename="systems.provided.example.hourly_with_order_simulator.yaml", ): @@ -33,10 +37,13 @@ def futures_system( sim_data = dbFuturesSimData() config = Config(config_filename) - + if use_limit_orders: + account = AccountWithOrderSimulatorForLimitOrders() + else: + account = AccountWithOrderSimulatorForHourlyMarketOrders() system = System( [ - AccountWithOrderSimulatorForHourlyMarketOrders(), + account, Portfolios(), PositionSizing(), ForecastCombine(), diff --git a/systems/provided/mr/accounts.py b/systems/provided/mr/accounts.py index 1f63bd0c3f..b0df7dc84e 100644 --- a/systems/provided/mr/accounts.py +++ b/systems/provided/mr/accounts.py @@ -1,8 +1,9 @@ import pandas as pd -from systems.system_cache import dont_cache, diagnostic, input -from systems.accounts.curves.account_curve import accountCurve -from systems.accounts.account_curve_order_simulator import AccountWithOrderSimulator +from systems.system_cache import diagnostic, input +from systems.accounts.order_simulator.account_curve_order_simulator import ( + AccountWithOrderSimulator, +) from systems.provided.mr.forecast_combine import MrForecastCombine from systems.provided.mr.rawdata import MrRawData diff --git a/systems/provided/mr/mr_pandl_order_simulator.py b/systems/provided/mr/mr_pandl_order_simulator.py index 8008faeee1..5bcc02691b 100644 --- a/systems/provided/mr/mr_pandl_order_simulator.py +++ b/systems/provided/mr/mr_pandl_order_simulator.py @@ -2,15 +2,10 @@ from dataclasses import dataclass import pandas as pd - -from syscore.cache import Cache -from systems.provided.mr.create_orders import DataForMROrder, create_orders_from_mr_data from systems.provided.mr.accounts import MrAccount -from systems.accounts.curves.account_curve import accountCurve -from systems.accounts.pandl_calculators.pandl_order_simulator import OrderSimulator -from systems.accounts.accounts_stage import Account -from sysobjects.orders import SimpleOrder, ListOfSimpleOrders +from systems.accounts.order_simulator.pandl_order_simulator import OrderSimulator +from sysobjects.orders import ListOfSimpleOrders @dataclass