From f3efda7042f76bdd67c6a4ecc1d32456528acc5d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 26 Jan 2024 21:35:50 +1100 Subject: [PATCH 01/42] Implement pyo3 instrument conversion methods --- nautilus_core/model/src/instruments/stubs.rs | 4 +- nautilus_trader/adapters/databento/data.py | 13 +-- nautilus_trader/model/instruments/__init__.py | 2 + nautilus_trader/model/instruments/base.pxd | 3 + nautilus_trader/model/instruments/base.pyx | 21 +++++ nautilus_trader/model/instruments/equity.pxd | 3 + nautilus_trader/model/instruments/equity.pyx | 35 ++++++++ .../model/instruments/futures_contract.pxd | 3 + .../model/instruments/futures_contract.pyx | 85 +++++++++++++------ .../model/instruments/options_contract.pxd | 3 + .../model/instruments/options_contract.pyx | 40 +++++++++ nautilus_trader/test_kit/providers.py | 2 +- .../test_kit/rust/instruments_pyo3.py | 4 +- .../model/instruments/test_equity_pyo3.py | 7 ++ .../instruments/test_futures_contract_pyo3.py | 11 ++- .../instruments/test_options_contract_pyo3.py | 7 ++ tests/unit_tests/model/test_instrument.py | 2 +- 17 files changed, 201 insertions(+), 44 deletions(-) diff --git a/nautilus_core/model/src/instruments/stubs.rs b/nautilus_core/model/src/instruments/stubs.rs index e3a6edff479b..487b4c9e9f9d 100644 --- a/nautilus_core/model/src/instruments/stubs.rs +++ b/nautilus_core/model/src/instruments/stubs.rs @@ -225,8 +225,8 @@ pub fn futures_contract_es() -> FuturesContract { let activation = Utc.with_ymd_and_hms(2021, 4, 8, 0, 0, 0).unwrap(); let expiration = Utc.with_ymd_and_hms(2021, 7, 8, 0, 0, 0).unwrap(); FuturesContract::new( - InstrumentId::new(Symbol::from("ESZ21"), Venue::from("CME")), - Symbol::from("ESZ21"), + InstrumentId::new(Symbol::from("ESZ1"), Venue::from("GLBX")), + Symbol::from("ESZ1"), AssetClass::Index, Ustr::from("ES"), activation.timestamp_nanos_opt().unwrap() as UnixNanos, diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index c97dfe392987..66e1004ee643 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -58,6 +58,7 @@ from nautilus_trader.model.instruments import FuturesContract from nautilus_trader.model.instruments import Instrument from nautilus_trader.model.instruments import OptionsContract +from nautilus_trader.model.instruments import instruments_from_pyo3 class DatabentoDataClient(LiveMarketDataClient): @@ -539,17 +540,7 @@ async def _request_instrument( end=(end or available_end).value, ) - instruments: list[Instrument] = [] - - for pyo3_instrument in pyo3_instruments: - if isinstance(pyo3_instrument, nautilus_pyo3.Equity): - instruments.append(Equity.from_dict(pyo3_instrument.to_dict())) - elif isinstance(pyo3_instrument, nautilus_pyo3.FuturesContract): - instruments.append(FuturesContract.from_dict(pyo3_instrument.to_dict())) - elif isinstance(pyo3_instrument, nautilus_pyo3.OptionsContract): - instruments.append(OptionsContract.from_dict(pyo3_instrument.to_dict())) - else: - self._log.warning(f"Instrument {pyo3_instrument} not supported") + instruments = instruments_from_pyo3(pyo3_instruments) self._handle_instruments( instruments=instruments, diff --git a/nautilus_trader/model/instruments/__init__.py b/nautilus_trader/model/instruments/__init__.py index fe92d379ec48..14ee79c70f45 100644 --- a/nautilus_trader/model/instruments/__init__.py +++ b/nautilus_trader/model/instruments/__init__.py @@ -18,6 +18,7 @@ """ from nautilus_trader.model.instruments.base import Instrument +from nautilus_trader.model.instruments.base import instruments_from_pyo3 from nautilus_trader.model.instruments.betting import BettingInstrument from nautilus_trader.model.instruments.crypto_future import CryptoFuture from nautilus_trader.model.instruments.crypto_perpetual import CryptoPerpetual @@ -38,4 +39,5 @@ "FuturesContract", "OptionsContract", "SyntheticInstrument", + "instruments_from_pyo3", ] diff --git a/nautilus_trader/model/instruments/base.pxd b/nautilus_trader/model/instruments/base.pxd index 913f8cbf529b..c55c8ddb01fb 100644 --- a/nautilus_trader/model/instruments/base.pxd +++ b/nautilus_trader/model/instruments/base.pxd @@ -97,3 +97,6 @@ cdef class Instrument(Data): cpdef Quantity make_qty(self, value) cpdef Money notional_value(self, Quantity quantity, Price price, bint use_quote_for_inverse=*) cpdef Quantity calculate_base_quantity(self, Quantity quantity, Price last_px) + + +cpdef list[Instrument] instruments_from_pyo3(list pyo3_instruments) diff --git a/nautilus_trader/model/instruments/base.pyx b/nautilus_trader/model/instruments/base.pyx index 9278d600e9bc..8c0616f0ce69 100644 --- a/nautilus_trader/model/instruments/base.pyx +++ b/nautilus_trader/model/instruments/base.pyx @@ -18,6 +18,8 @@ from decimal import Decimal from libc.math cimport pow from libc.stdint cimport uint64_t +from nautilus_trader.core import nautilus_pyo3 + from nautilus_trader.core.correctness cimport Condition from nautilus_trader.core.rust.model cimport AssetClass from nautilus_trader.core.rust.model cimport InstrumentClass @@ -26,6 +28,9 @@ from nautilus_trader.model.functions cimport asset_class_to_str from nautilus_trader.model.functions cimport instrument_class_from_str from nautilus_trader.model.functions cimport instrument_class_to_str from nautilus_trader.model.identifiers cimport InstrumentId +from nautilus_trader.model.instruments.equity cimport Equity +from nautilus_trader.model.instruments.futures_contract cimport FuturesContract +from nautilus_trader.model.instruments.options_contract cimport OptionsContract from nautilus_trader.model.objects cimport Currency from nautilus_trader.model.objects cimport Quantity from nautilus_trader.model.tick_scheme.base cimport TICK_SCHEMES @@ -548,3 +553,19 @@ cdef class Instrument(Data): Condition.not_none(quantity, "quantity") return Quantity(quantity.as_f64_c() * (1.0 / last_px.as_f64_c()), self.size_precision) + + +cpdef list[Instrument] instruments_from_pyo3(list pyo3_instruments): + cdef list[Instrument] instruments = [] + + for pyo3_instrument in pyo3_instruments: + if isinstance(pyo3_instrument, nautilus_pyo3.Equity): + instruments.append(Equity.from_pyo3_c(pyo3_instrument)) + elif isinstance(pyo3_instrument, nautilus_pyo3.FuturesContract): + instruments.append(FuturesContract.from_pyo3_c(pyo3_instrument)) + elif isinstance(pyo3_instrument, nautilus_pyo3.OptionsContract): + instruments.append(OptionsContract.from_pyo3_c(pyo3_instrument)) + else: + RuntimeError(f"Instrument {pyo3_instrument} not supported") + + return instruments diff --git a/nautilus_trader/model/instruments/equity.pxd b/nautilus_trader/model/instruments/equity.pxd index 5440d1ad4659..ecbae20f7657 100644 --- a/nautilus_trader/model/instruments/equity.pxd +++ b/nautilus_trader/model/instruments/equity.pxd @@ -24,3 +24,6 @@ cdef class Equity(Instrument): @staticmethod cdef dict to_dict_c(Equity obj) + + @staticmethod + cdef Equity from_pyo3_c(pyo3_instrument) diff --git a/nautilus_trader/model/instruments/equity.pyx b/nautilus_trader/model/instruments/equity.pyx index 4702e61e25d9..2e64f9f8e480 100644 --- a/nautilus_trader/model/instruments/equity.pyx +++ b/nautilus_trader/model/instruments/equity.pyx @@ -158,6 +158,24 @@ cdef class Equity(Instrument): "ts_init": obj.ts_init, } + @staticmethod + cdef Equity from_pyo3_c(pyo3_instrument): + return Equity( + instrument_id=InstrumentId.from_str_c(pyo3_instrument.id.value), + raw_symbol=Symbol(pyo3_instrument.id.symbol.value), + currency=Currency.from_str_c(pyo3_instrument.currency.code), + price_precision=pyo3_instrument.price_precision, + price_increment=Price.from_raw_c(pyo3_instrument.price_increment.raw, pyo3_instrument.price_precision), + lot_size=Quantity.from_raw_c(pyo3_instrument.lot_size.raw, pyo3_instrument.lot_size.precision), + isin=pyo3_instrument.isin, + margin_init=None, # None for now + margin_maint=None, # None for now + maker_fee=None, # None for now + taker_fee=None, # None for now + ts_event=pyo3_instrument.ts_event, + ts_init=pyo3_instrument.ts_init, + ) + @staticmethod def from_dict(dict values) -> Instrument: """ @@ -186,3 +204,20 @@ cdef class Equity(Instrument): """ return Equity.to_dict_c(obj) + + @staticmethod + def from_pyo3(pyo3_instrument) -> Equity: + """ + Return legacy Cython equity instrument converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_instrument : nautilus_pyo3.Equity + The pyo3 Rust equity instrument to convert from. + + Returns + ------- + Equity + + """ + return Equity.from_pyo3_c(pyo3_instrument) diff --git a/nautilus_trader/model/instruments/futures_contract.pxd b/nautilus_trader/model/instruments/futures_contract.pxd index 0d53e4095904..5b3aaa3aae1a 100644 --- a/nautilus_trader/model/instruments/futures_contract.pxd +++ b/nautilus_trader/model/instruments/futures_contract.pxd @@ -31,3 +31,6 @@ cdef class FuturesContract(Instrument): @staticmethod cdef dict to_dict_c(FuturesContract obj) + + @staticmethod + cdef FuturesContract from_pyo3_c(pyo3_instrument) diff --git a/nautilus_trader/model/instruments/futures_contract.pyx b/nautilus_trader/model/instruments/futures_contract.pyx index 52eb75da8bc9..7726ac29f36a 100644 --- a/nautilus_trader/model/instruments/futures_contract.pyx +++ b/nautilus_trader/model/instruments/futures_contract.pyx @@ -153,6 +153,32 @@ cdef class FuturesContract(Instrument): f"info={self.info})" ) + @property + def activation_utc(self) -> pd.Timestamp: + """ + Return the contract activation timestamp (UTC). + + Returns + ------- + pd.Timestamp + tz-aware UTC. + + """ + return pd.Timestamp(self.activation_ns, tz=pytz.utc) + + @property + def expiration_utc(self) -> pd.Timestamp: + """ + Return the contract expriation timestamp (UTC). + + Returns + ------- + pd.Timestamp + tz-aware UTC. + + """ + return pd.Timestamp(self.expiration_ns, tz=pytz.utc) + @staticmethod cdef FuturesContract from_dict_c(dict values): Condition.not_none(values, "values") @@ -196,31 +222,23 @@ cdef class FuturesContract(Instrument): "ts_init": obj.ts_init, } - @property - def activation_utc(self) -> pd.Timestamp: - """ - Return the contract activation timestamp (UTC). - - Returns - ------- - pd.Timestamp - tz-aware UTC. - - """ - return pd.Timestamp(self.activation_ns, tz=pytz.utc) - - @property - def expiration_utc(self) -> pd.Timestamp: - """ - Return the contract expriation timestamp (UTC). - - Returns - ------- - pd.Timestamp - tz-aware UTC. - - """ - return pd.Timestamp(self.expiration_ns, tz=pytz.utc) + @staticmethod + cdef FuturesContract from_pyo3_c(pyo3_instrument): + return FuturesContract( + instrument_id=InstrumentId.from_str_c(pyo3_instrument.id.value), + raw_symbol=Symbol(pyo3_instrument.raw_symbol.value), + asset_class=asset_class_from_str(str(pyo3_instrument.asset_class)), + currency=Currency.from_str_c(pyo3_instrument.currency.code), + price_precision=pyo3_instrument.price_precision, + price_increment=Price.from_raw_c(pyo3_instrument.price_increment.raw, pyo3_instrument.price_precision), + multiplier=Quantity.from_raw_c(pyo3_instrument.multiplier.raw, 0), + lot_size=Quantity.from_raw_c(pyo3_instrument.lot_size.raw, 0), + underlying=pyo3_instrument.underlying, + activation_ns=pyo3_instrument.activation_ns, + expiration_ns=pyo3_instrument.expiration_ns, + ts_event=pyo3_instrument.ts_event, + ts_init=pyo3_instrument.ts_init, + ) @staticmethod def from_dict(dict values) -> FuturesContract: @@ -250,3 +268,20 @@ cdef class FuturesContract(Instrument): """ return FuturesContract.to_dict_c(obj) + + @staticmethod + def from_pyo3(pyo3_instrument) -> FuturesContract: + """ + Return legacy Cython futures contract instrument converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_instrument : nautilus_pyo3.FuturesContract + The pyo3 Rust futures contract instrument to convert from. + + Returns + ------- + FuturesContract + + """ + return FuturesContract.from_pyo3_c(pyo3_instrument) diff --git a/nautilus_trader/model/instruments/options_contract.pxd b/nautilus_trader/model/instruments/options_contract.pxd index 5a85591f44d6..8658bbbe4e5b 100644 --- a/nautilus_trader/model/instruments/options_contract.pxd +++ b/nautilus_trader/model/instruments/options_contract.pxd @@ -37,3 +37,6 @@ cdef class OptionsContract(Instrument): @staticmethod cdef dict to_dict_c(OptionsContract obj) + + @staticmethod + cdef OptionsContract from_pyo3_c(pyo3_instrument) diff --git a/nautilus_trader/model/instruments/options_contract.pyx b/nautilus_trader/model/instruments/options_contract.pyx index 682f64e2fdce..fbb1d85b11bd 100644 --- a/nautilus_trader/model/instruments/options_contract.pyx +++ b/nautilus_trader/model/instruments/options_contract.pyx @@ -166,6 +166,8 @@ cdef class OptionsContract(Instrument): """ return pd.Timestamp(self.expiration_ns, tz=pytz.utc) + + @staticmethod cdef OptionsContract from_dict_c(dict values): Condition.not_none(values, "values") @@ -213,6 +215,27 @@ cdef class OptionsContract(Instrument): "ts_init": obj.ts_init, } + @staticmethod + cdef OptionsContract from_pyo3_c(pyo3_instrument): + Condition.not_none(pyo3_instrument, "pyo3_instrument") + return OptionsContract( + instrument_id=InstrumentId.from_str_c(pyo3_instrument.id.value), + raw_symbol=Symbol(pyo3_instrument.raw_symbol.value), + asset_class=asset_class_from_str(str(pyo3_instrument.asset_class)), + currency=Currency.from_str_c(pyo3_instrument.currency.code), + price_precision=pyo3_instrument.price_precision, + price_increment=Price.from_raw_c(pyo3_instrument.price_increment.raw, pyo3_instrument.price_precision), + multiplier=Quantity.from_raw_c(pyo3_instrument.multiplier.raw, 0), + lot_size=Quantity.from_raw_c(pyo3_instrument.lot_size.raw, 0), + underlying=pyo3_instrument.underlying, + option_kind=option_kind_from_str(str(pyo3_instrument.option_kind)), + activation_ns=pyo3_instrument.activation_ns, + expiration_ns=pyo3_instrument.expiration_ns, + strike_price=Price.from_raw_c(pyo3_instrument.strike_price.raw, pyo3_instrument.strike_price.precision), + ts_event=pyo3_instrument.ts_event, + ts_init=pyo3_instrument.ts_init, + ) + @staticmethod def from_dict(dict values) -> OptionsContract: """ @@ -241,3 +264,20 @@ cdef class OptionsContract(Instrument): """ return OptionsContract.to_dict_c(obj) + + @staticmethod + def from_pyo3(pyo3_instrument) -> OptionsContract: + """ + Return legacy Cython options contract instrument converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_instrument : nautilus_pyo3.OptionsContract + The pyo3 Rust options contract instrument to convert from. + + Returns + ------- + OptionsContract + + """ + return OptionsContract.from_pyo3_c(pyo3_instrument) diff --git a/nautilus_trader/test_kit/providers.py b/nautilus_trader/test_kit/providers.py index abafcc134589..ab885d6b3be8 100644 --- a/nautilus_trader/test_kit/providers.py +++ b/nautilus_trader/test_kit/providers.py @@ -498,7 +498,7 @@ def es_future( @staticmethod def future( - symbol: str = "ESZ21", + symbol: str = "ESZ1", underlying: str = "ES", venue: str = "GLBX", ) -> FuturesContract: diff --git a/nautilus_trader/test_kit/rust/instruments_pyo3.py b/nautilus_trader/test_kit/rust/instruments_pyo3.py index 4a2aa21b169c..47b8d6dfbdd6 100644 --- a/nautilus_trader/test_kit/rust/instruments_pyo3.py +++ b/nautilus_trader/test_kit/rust/instruments_pyo3.py @@ -219,8 +219,8 @@ def futures_contract_es( if expiration is None: expiration = pd.Timestamp("2021-12-17", tz=pytz.utc) return FuturesContract( - id=InstrumentId.from_str("ESZ21.CME"), - raw_symbol=Symbol("ESZ21"), + id=InstrumentId.from_str("ESZ1.GLBX"), + raw_symbol=Symbol("ESZ1"), asset_class=AssetClass.INDEX, underlying="ES", activation_ns=activation.value, diff --git a/tests/unit_tests/model/instruments/test_equity_pyo3.py b/tests/unit_tests/model/instruments/test_equity_pyo3.py index 6d6b1dccdebb..f39cf00454a3 100644 --- a/tests/unit_tests/model/instruments/test_equity_pyo3.py +++ b/tests/unit_tests/model/instruments/test_equity_pyo3.py @@ -15,6 +15,7 @@ from nautilus_trader.core.nautilus_pyo3 import Equity from nautilus_trader.core.nautilus_pyo3 import InstrumentId +from nautilus_trader.model.instruments import Equity as LegacyEquity from nautilus_trader.test_kit.rust.instruments_pyo3 import TestInstrumentProviderPyo3 @@ -54,3 +55,9 @@ def test_to_dict(): "ts_event": 0, "ts_init": 0, } + + +def test_legacy_equity_from_pyo3(): + equity = LegacyEquity.from_pyo3(_AAPL_EQUITY) + + assert equity.id.value == "AAPL.XNAS" diff --git a/tests/unit_tests/model/instruments/test_futures_contract_pyo3.py b/tests/unit_tests/model/instruments/test_futures_contract_pyo3.py index 7d06c16c8b46..9d12f0ccaf98 100644 --- a/tests/unit_tests/model/instruments/test_futures_contract_pyo3.py +++ b/tests/unit_tests/model/instruments/test_futures_contract_pyo3.py @@ -14,6 +14,7 @@ # ------------------------------------------------------------------------------------------------- from nautilus_trader.core.nautilus_pyo3 import FuturesContract +from nautilus_trader.model.instruments import FuturesContract as LegacyFuturesContract from nautilus_trader.test_kit.rust.instruments_pyo3 import TestInstrumentProviderPyo3 @@ -35,8 +36,8 @@ def test_to_dict(): assert FuturesContract.from_dict(result) == _ES_FUTURE assert result == { "type": "FuturesContract", - "id": "ESZ21.CME", - "raw_symbol": "ESZ21", + "id": "ESZ1.GLBX", + "raw_symbol": "ESZ1", "asset_class": "INDEX", "underlying": "ES", "activation_ns": 1631836800000000000, @@ -53,3 +54,9 @@ def test_to_dict(): "ts_event": 0, "ts_init": 0, } + + +def test_legacy_futures_contract_from_pyo3(): + future = LegacyFuturesContract.from_pyo3(_ES_FUTURE) + + assert future.id.value == "ESZ1.GLBX" diff --git a/tests/unit_tests/model/instruments/test_options_contract_pyo3.py b/tests/unit_tests/model/instruments/test_options_contract_pyo3.py index fbdc2d77c532..be84b41d7664 100644 --- a/tests/unit_tests/model/instruments/test_options_contract_pyo3.py +++ b/tests/unit_tests/model/instruments/test_options_contract_pyo3.py @@ -14,6 +14,7 @@ # ------------------------------------------------------------------------------------------------- from nautilus_trader.core.nautilus_pyo3 import OptionsContract +from nautilus_trader.model.instruments import OptionsContract as LegacyOptionsContract from nautilus_trader.test_kit.rust.instruments_pyo3 import TestInstrumentProviderPyo3 @@ -55,3 +56,9 @@ def test_to_dict(): "ts_event": 0, "ts_init": 0, } + + +def test_legacy_options_contract_from_pyo3(): + option = LegacyOptionsContract.from_pyo3(_AAPL_OPTION) + + assert option.id.value == "AAPL211217C00150000.OPRA" diff --git a/tests/unit_tests/model/test_instrument.py b/tests/unit_tests/model/test_instrument.py index 698c574fc13f..7ddef288e4b5 100644 --- a/tests/unit_tests/model/test_instrument.py +++ b/tests/unit_tests/model/test_instrument.py @@ -473,7 +473,7 @@ def test_pyo3_future_to_legacy_future() -> None: # Assert assert isinstance(instrument, FuturesContract) - assert instrument.id == InstrumentId.from_str("ESZ21.CME") + assert instrument.id == InstrumentId.from_str("ESZ1.GLBX") def test_pyo3_option_to_legacy_option() -> None: From 1f3d17f403935579d4e123f6fda9295fcb9be053 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 26 Jan 2024 21:44:47 +1100 Subject: [PATCH 02/42] Bump version --- RELEASES.md | 15 +++++++++++++++ nautilus_core/Cargo.lock | 20 ++++++++++---------- nautilus_core/Cargo.toml | 2 +- poetry.lock | 4 ++-- pyproject.toml | 2 +- version.json | 2 +- 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 11a0f000bcd2..91bda009b91c 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,18 @@ +# NautilusTrader 1.186.0 Beta + +Released on TBD (UTC). + +### Enhancements +None + +### Breaking Changes +None + +### Fixes +None + +--- + # NautilusTrader 1.185.0 Beta Released on 26th January 2024 (UTC). diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index fe7951cf1e1b..466478219b1c 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2415,7 +2415,7 @@ dependencies = [ [[package]] name = "nautilus-adapters" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "chrono", @@ -2444,7 +2444,7 @@ dependencies = [ [[package]] name = "nautilus-backtest" -version = "0.16.0" +version = "0.17.0" dependencies = [ "cbindgen", "nautilus-common", @@ -2458,7 +2458,7 @@ dependencies = [ [[package]] name = "nautilus-common" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "cbindgen", @@ -2482,7 +2482,7 @@ dependencies = [ [[package]] name = "nautilus-core" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "cbindgen", @@ -2501,7 +2501,7 @@ dependencies = [ [[package]] name = "nautilus-indicators" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "nautilus-core", @@ -2513,7 +2513,7 @@ dependencies = [ [[package]] name = "nautilus-infrastructure" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "nautilus-common", @@ -2528,7 +2528,7 @@ dependencies = [ [[package]] name = "nautilus-model" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "cbindgen", @@ -2556,7 +2556,7 @@ dependencies = [ [[package]] name = "nautilus-network" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "axum", @@ -2581,7 +2581,7 @@ dependencies = [ [[package]] name = "nautilus-persistence" -version = "0.16.0" +version = "0.17.0" dependencies = [ "anyhow", "binary-heap-plus", @@ -2605,7 +2605,7 @@ dependencies = [ [[package]] name = "nautilus-pyo3" -version = "0.16.0" +version = "0.17.0" dependencies = [ "nautilus-adapters", "nautilus-common", diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index e2a1280e9f5d..0c90abc1a69e 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -16,7 +16,7 @@ members = [ [workspace.package] rust-version = "1.75.0" -version = "0.16.0" +version = "0.17.0" edition = "2021" authors = ["Nautech Systems "] description = "A high-performance algorithmic trading platform and event-driven backtester" diff --git a/poetry.lock b/poetry.lock index 80bca1f409d0..1144fe3c8a50 100644 --- a/poetry.lock +++ b/poetry.lock @@ -528,7 +528,7 @@ name = "css-html-js-minify" version = "2.5.5" description = "CSS HTML JS Minifier" optional = false -python-versions = ">=3.6" +python-versions = "*" files = [ {file = "css-html-js-minify-2.5.5.zip", hash = "sha256:4a9f11f7e0496f5284d12111f3ba4ff5ff2023d12f15d195c9c48bd97013746c"}, {file = "css_html_js_minify-2.5.5-py2.py3-none-any.whl", hash = "sha256:3da9d35ac0db8ca648c1b543e0e801d7ca0bab9e6bfd8418fee59d5ae001727a"}, @@ -1764,7 +1764,7 @@ name = "pycparser" version = "2.21" description = "C parser in Python" optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = "*" files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, diff --git a/pyproject.toml b/pyproject.toml index 0ac9af46a3b7..ce35e4d7524a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nautilus_trader" -version = "1.185.0" +version = "1.186.0" description = "A high-performance algorithmic trading platform and event-driven backtester" authors = ["Nautech Systems "] license = "LGPL-3.0-or-later" diff --git a/version.json b/version.json index 6f0efd7860e1..7338309232ed 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, "label": "", - "message": "v1.185.0", + "message": "v1.186.0", "color": "orange" } From a0a0bf8bb0d99475c9d61f14f30094a37f7219fc Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 07:27:41 +1100 Subject: [PATCH 03/42] Update dependencies --- nautilus_core/Cargo.lock | 4 +- poetry.lock | 112 +++++++++++++++++++-------------------- pyproject.toml | 2 +- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 466478219b1c..b505662248cb 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -5397,9 +5397,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.34" +version = "0.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" +checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" dependencies = [ "memchr", ] diff --git a/poetry.lock b/poetry.lock index 1144fe3c8a50..cd4d9f067f5f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -458,63 +458,63 @@ files = [ [[package]] name = "coverage" -version = "7.4.0" +version = "7.4.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, - {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, - {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, - {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, - {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, - {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, - {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, - {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, - {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, - {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, - {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, - {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, - {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, - {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, - {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, - {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, - {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, - {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, - {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, - {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, - {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, - {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, - {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, - {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, - {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, - {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, - {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, - {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, - {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, - {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, - {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, - {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, - {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, - {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, - {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, - {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, - {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, - {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, - {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, - {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, - {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, - {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, - {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, - {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, - {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, - {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, - {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, - {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, - {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, - {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, - {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, - {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:077d366e724f24fc02dbfe9d946534357fda71af9764ff99d73c3c596001bbd7"}, + {file = "coverage-7.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0193657651f5399d433c92f8ae264aff31fc1d066deee4b831549526433f3f61"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17bbc946f52ca67adf72a5ee783cd7cd3477f8f8796f59b4974a9b59cacc9ee"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3277f5fa7483c927fe3a7b017b39351610265308f5267ac6d4c2b64cc1d8d25"}, + {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dceb61d40cbfcf45f51e59933c784a50846dc03211054bd76b421a713dcdf19"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6008adeca04a445ea6ef31b2cbaf1d01d02986047606f7da266629afee982630"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c61f66d93d712f6e03369b6a7769233bfda880b12f417eefdd4f16d1deb2fc4c"}, + {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9bb62fac84d5f2ff523304e59e5c439955fb3b7f44e3d7b2085184db74d733b"}, + {file = "coverage-7.4.1-cp310-cp310-win32.whl", hash = "sha256:f86f368e1c7ce897bf2457b9eb61169a44e2ef797099fb5728482b8d69f3f016"}, + {file = "coverage-7.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:869b5046d41abfea3e381dd143407b0d29b8282a904a19cb908fa24d090cc018"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8ffb498a83d7e0305968289441914154fb0ef5d8b3157df02a90c6695978295"}, + {file = "coverage-7.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cacfaefe6089d477264001f90f55b7881ba615953414999c46cc9713ff93c8c"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6850e6e36e332d5511a48a251790ddc545e16e8beaf046c03985c69ccb2676"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e961aa13b6d47f758cc5879383d27b5b3f3dcd9ce8cdbfdc2571fe86feb4dd"}, + {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfd1e1b9f0898817babf840b77ce9fe655ecbe8b1b327983df485b30df8cc011"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b00e21f86598b6330f0019b40fb397e705135040dbedc2ca9a93c7441178e74"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:536d609c6963c50055bab766d9951b6c394759190d03311f3e9fcf194ca909e1"}, + {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ac8f8eb153724f84885a1374999b7e45734bf93a87d8df1e7ce2146860edef6"}, + {file = "coverage-7.4.1-cp311-cp311-win32.whl", hash = "sha256:f3771b23bb3675a06f5d885c3630b1d01ea6cac9e84a01aaf5508706dba546c5"}, + {file = "coverage-7.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d2f9d4cc2a53b38cabc2d6d80f7f9b7e3da26b2f53d48f05876fef7956b6968"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f68ef3660677e6624c8cace943e4765545f8191313a07288a53d3da188bd8581"}, + {file = "coverage-7.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23b27b8a698e749b61809fb637eb98ebf0e505710ec46a8aa6f1be7dc0dc43a6"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3424c554391dc9ef4a92ad28665756566a28fecf47308f91841f6c49288e66"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0860a348bf7004c812c8368d1fc7f77fe8e4c095d661a579196a9533778e156"}, + {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe558371c1bdf3b8fa03e097c523fb9645b8730399c14fe7721ee9c9e2a545d3"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3468cc8720402af37b6c6e7e2a9cdb9f6c16c728638a2ebc768ba1ef6f26c3a1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:02f2edb575d62172aa28fe00efe821ae31f25dc3d589055b3fb64d51e52e4ab1"}, + {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ca6e61dc52f601d1d224526360cdeab0d0712ec104a2ce6cc5ccef6ed9a233bc"}, + {file = "coverage-7.4.1-cp312-cp312-win32.whl", hash = "sha256:ca7b26a5e456a843b9b6683eada193fc1f65c761b3a473941efe5a291f604c74"}, + {file = "coverage-7.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:85ccc5fa54c2ed64bd91ed3b4a627b9cce04646a659512a051fa82a92c04a448"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bdb0285a0202888d19ec6b6d23d5990410decb932b709f2b0dfe216d031d218"}, + {file = "coverage-7.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:918440dea04521f499721c039863ef95433314b1db00ff826a02580c1f503e45"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379d4c7abad5afbe9d88cc31ea8ca262296480a86af945b08214eb1a556a3e4d"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b094116f0b6155e36a304ff912f89bbb5067157aff5f94060ff20bbabdc8da06"}, + {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f5968608b1fe2a1d00d01ad1017ee27efd99b3437e08b83ded9b7af3f6f766"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:10e88e7f41e6197ea0429ae18f21ff521d4f4490aa33048f6c6f94c6045a6a75"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a4a3907011d39dbc3e37bdc5df0a8c93853c369039b59efa33a7b6669de04c60"}, + {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d224f0c4c9c98290a6990259073f496fcec1b5cc613eecbd22786d398ded3ad"}, + {file = "coverage-7.4.1-cp38-cp38-win32.whl", hash = "sha256:23f5881362dcb0e1a92b84b3c2809bdc90db892332daab81ad8f642d8ed55042"}, + {file = "coverage-7.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:a07f61fc452c43cd5328b392e52555f7d1952400a1ad09086c4a8addccbd138d"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e738a492b6221f8dcf281b67129510835461132b03024830ac0e554311a5c54"}, + {file = "coverage-7.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46342fed0fff72efcda77040b14728049200cbba1279e0bf1188f1f2078c1d70"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9641e21670c68c7e57d2053ddf6c443e4f0a6e18e547e86af3fad0795414a628"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb2c2688ed93b027eb0d26aa188ada34acb22dceea256d76390eea135083950"}, + {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12c923757de24e4e2110cf8832d83a886a4cf215c6e61ed506006872b43a6d1"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0491275c3b9971cdbd28a4595c2cb5838f08036bca31765bad5e17edf900b2c7"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8dfc5e195bbef80aabd81596ef52a1277ee7143fe419efc3c4d8ba2754671756"}, + {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a78b656a4d12b0490ca72651fe4d9f5e07e3c6461063a9b6265ee45eb2bdd35"}, + {file = "coverage-7.4.1-cp39-cp39-win32.whl", hash = "sha256:f90515974b39f4dea2f27c0959688621b46d96d5a626cf9c53dbc653a895c05c"}, + {file = "coverage-7.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:64e723ca82a84053dd7bfcc986bdb34af8d9da83c521c19d6b472bc6880e191a"}, + {file = "coverage-7.4.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:32a8d985462e37cfdab611a6f95b09d7c091d07668fdc26e47a725ee575fe166"}, + {file = "coverage-7.4.1.tar.gz", hash = "sha256:1ed4b95480952b1a26d863e546fa5094564aa0065e1e5f0d4d0041f293251d04"}, ] [package.dependencies] @@ -528,7 +528,7 @@ name = "css-html-js-minify" version = "2.5.5" description = "CSS HTML JS Minifier" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ {file = "css-html-js-minify-2.5.5.zip", hash = "sha256:4a9f11f7e0496f5284d12111f3ba4ff5ff2023d12f15d195c9c48bd97013746c"}, {file = "css_html_js_minify-2.5.5-py2.py3-none-any.whl", hash = "sha256:3da9d35ac0db8ca648c1b543e0e801d7ca0bab9e6bfd8418fee59d5ae001727a"}, @@ -1764,7 +1764,7 @@ name = "pycparser" version = "2.21" description = "C parser in Python" optional = true -python-versions = "*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -2801,4 +2801,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "d6d023fecc2788959a310c6b9c1a44a9d82059af5f2aae009f9e1358e7afda54" +content-hash = "132915d961ce55fc8cd96ad424fade816e9a4e9a54bbd7cd2817424c1d21b706" diff --git a/pyproject.toml b/pyproject.toml index ce35e4d7524a..9edb5e885690 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ types-toml = "^0.10.2" optional = true [tool.poetry.group.test.dependencies] -coverage = "^7.4.0" +coverage = "^7.4.1" pytest = "^7.4.4" pytest-aiohttp = "^1.0.5" pytest-asyncio = "==0.21.1" # Pinned due Cython: cannot set '__pytest_asyncio_scoped_event_loop' attribute of immutable type From c0744037f75476930d5e4d706abad61eb2ddd075 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 08:15:29 +1100 Subject: [PATCH 04/42] Improve DatabentoDataClient historical requests --- nautilus_trader/adapters/databento/data.py | 154 +++++++++++++-------- 1 file changed, 93 insertions(+), 61 deletions(-) diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 66e1004ee643..bd2c674daced 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -54,10 +54,6 @@ from nautilus_trader.model.enums import bar_aggregation_to_str from nautilus_trader.model.identifiers import InstrumentId from nautilus_trader.model.identifiers import Venue -from nautilus_trader.model.instruments import Equity -from nautilus_trader.model.instruments import FuturesContract -from nautilus_trader.model.instruments import Instrument -from nautilus_trader.model.instruments import OptionsContract from nautilus_trader.model.instruments import instruments_from_pyo3 @@ -132,7 +128,8 @@ def __init__( self._live_clients_mbo: dict[Dataset, databento.Live] = {} self._has_subscribed: dict[Dataset, bool] = {} self._loader = loader or DatabentoDataLoader() - self._available_ends: dict[Dataset, pd.Timestamp] = {} + self._dataset_ranges: dict[Dataset, tuple[pd.Timestamp, pd.Timestamp]] = {} + self._dataset_ranges_requested: set[Dataset] = set() # Cache instrument index for instrument_id in config.instrument_ids or []: @@ -145,6 +142,10 @@ def __init__( self._buffered_mbo_subscriptions: dict[Dataset, list[InstrumentId]] = defaultdict(list) self._buffered_deltas: dict[InstrumentId, list[OrderBookDelta]] = defaultdict(list) + # Tasks + self._update_dataset_ranges_interval_seconds: int = 60 * 60 # Once per hour (hard coded) + self._update_dataset_ranges_task: asyncio.Task | None = None + async def _connect(self) -> None: if not self._instrument_ids: return # Nothing else to do yet @@ -169,6 +170,7 @@ async def _connect(self) -> None: self._log.warning("Timeout waiting for instruments...") self._send_all_instruments_to_data_engine() + self._update_dataset_ranges_task = self.create_task(self._update_dataset_ranges()) async def _disconnect(self) -> None: coros: list[Coroutine] = [] @@ -187,8 +189,35 @@ async def _disconnect(self) -> None: self._buffer_mbo_subscriptions_task.cancel() self._buffer_mbo_subscriptions_task = None + # Cancel update dataset ranges task + if self._update_dataset_ranges_task: + self._log.debug("Canceling `update_dataset_ranges` task...") + self._update_dataset_ranges_task.cancel() + self._update_dataset_ranges_task = None + await asyncio.gather(*coros) + async def _update_dataset_ranges(self) -> None: + while True: + try: + self._log.debug( + f"Scheduled `update_instruments` to run in " + f"{self._update_dataset_ranges_interval_seconds}s.", + ) + + await asyncio.sleep(self._update_dataset_ranges_interval_seconds) + + tasks = [] + for dataset in self._dataset_ranges: + tasks.append(self._get_dataset_range(dataset)) + + await asyncio.gather(*tasks) + except Exception as e: # Create specific exception type + self._log.error(f"Error updating dataset range: {e}") + except asyncio.CancelledError: + self._log.debug("Canceled `update_dataset_ranges` task.") + break + async def _buffer_mbo_subscriptions(self) -> None: try: await asyncio.sleep(self._mbo_subscriptions_delay or 0.0) @@ -253,15 +282,40 @@ async def _ensure_subscribed_for_instrument(self, instrument_id: InstrumentId) - self._instrument_ids[dataset].add(instrument_id) await self._subscribe_instrument(instrument_id) - async def _get_dataset_range(self, dataset: Dataset) -> tuple[pd.Timestamp, pd.Timestamp]: - response = await self._http_client.get_dataset_range(dataset) + async def _get_dataset_range( + self, + dataset: Dataset, + ) -> tuple[pd.Timestamp | None, pd.Timestamp]: + # Check and cache dataset available range + while dataset in self._dataset_ranges_requested: + await asyncio.sleep(0.1) - start = pd.Timestamp(response["start_date"], tz=pytz.utc) - end = pd.Timestamp(response["end_date"], tz=pytz.utc) + available_range = self._dataset_ranges.get(dataset) + if available_range: + return available_range - self._log.info(f"Dataset {dataset} available end {end}.", LogColor.BLUE) + self._dataset_ranges_requested.add(dataset) - return start, end + try: + self._log.info(f"Requesting dataset range for {dataset}...", LogColor.BLUE) + response = await self._http_client.get_dataset_range(dataset) + + available_start = pd.Timestamp(response["start_date"], tz=pytz.utc) + available_end = pd.Timestamp(response["end_date"], tz=pytz.utc) + + self._dataset_ranges[dataset] = (available_start, available_end) + + self._log.info( + f"Dataset {dataset} available end {available_end.date()}.", + LogColor.BLUE, + ) + + return available_start, available_end + except Exception as e: # More specific exception + self._log.error(f"Error requesting dataset range: {e}") + return (None, pd.Timestamp.utcnow()) + finally: + self._dataset_ranges_requested.discard(dataset) # -- OVERRIDES ---------------------------------------------------------------------------- @@ -517,27 +571,22 @@ async def _request_instrument( end: pd.Timestamp | None = None, ) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - - # Check and cache dataset available end - available_end = self._available_ends.get(dataset) - if available_end is None: - _, available_end = await self._get_dataset_range(dataset) - self._available_ends[dataset] = available_end + _, available_end = await self._get_dataset_range(dataset) start = start or available_end - ONE_DAY * 2 end = end or available_end self._log.info( f"Requesting {instrument_id} instrument definitions: " - f"dataset={dataset}, start={start}, end={end} ...", + f"dataset={dataset}, start={start}, end={end}", LogColor.BLUE, ) pyo3_instruments = await self._http_client.get_range_instruments( dataset=dataset, symbols=ALL_SYMBOLS, - start=(start or available_end - ONE_DAY * 2).value, - end=(end or available_end).value, + start=start.value, + end=end.value, ) instruments = instruments_from_pyo3(pyo3_instruments) @@ -555,19 +604,14 @@ async def _request_instruments( end: pd.Timestamp | None = None, ) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(venue) - - # Check and cache dataset available end - available_end = self._available_ends.get(dataset) - if available_end is None: - _, available_end = await self._get_dataset_range(dataset) - self._available_ends[dataset] = available_end + _, available_end = await self._get_dataset_range(dataset) start = start or available_end - ONE_DAY * 2 end = end or available_end self._log.info( f"Requesting {venue} instrument definitions: " - f"dataset={dataset}, start={start}, end={end} ...", + f"dataset={dataset}, start={start}, end={end}", LogColor.BLUE, ) @@ -578,17 +622,7 @@ async def _request_instruments( end=end.value, ) - instruments: list[Instrument] = [] - - for pyo3_instrument in pyo3_instruments: - if isinstance(pyo3_instrument, nautilus_pyo3.Equity): - instruments.append(Equity.from_dict(pyo3_instrument.to_dict())) - elif isinstance(pyo3_instrument, nautilus_pyo3.FuturesContract): - instruments.append(FuturesContract.from_dict(pyo3_instrument.to_dict())) - elif isinstance(pyo3_instrument, nautilus_pyo3.OptionsContract): - instruments.append(OptionsContract.from_dict(pyo3_instrument.to_dict())) - else: - self._log.warning(f"Instrument {pyo3_instrument} not supported") + instruments = instruments_from_pyo3(pyo3_instruments) self._handle_instruments( instruments=instruments, @@ -605,19 +639,19 @@ async def _request_quote_ticks( end: pd.Timestamp | None = None, ) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - - # Check and cache dataset available end - available_end = self._available_ends.get(dataset) - if available_end is None: - _, available_end = await self._get_dataset_range(dataset) - self._available_ends[dataset] = available_end + _, available_end = await self._get_dataset_range(dataset) start = start or available_end - ONE_DAY end = end or available_end + if limit > 0: + self._log.warning( + f"Ignoring limit {limit} because its applied from the start (instead of the end).", + ) + self._log.info( f"Requesting {instrument_id} quote ticks: " - f"dataset={dataset}, start={start}, end={end}, limit={limit} ...", + f"dataset={dataset}, start={start}, end={end}", LogColor.BLUE, ) @@ -626,7 +660,6 @@ async def _request_quote_ticks( symbols=instrument_id.symbol.value, start=start.value, end=end.value, - limit=limit, ) quotes = QuoteTick.from_pyo3_list(pyo3_quotes) @@ -646,19 +679,19 @@ async def _request_trade_ticks( end: pd.Timestamp | None = None, ) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - - # Check and cache dataset available end - available_end = self._available_ends.get(dataset) - if available_end is None: - _, available_end = await self._get_dataset_range(dataset) - self._available_ends[dataset] = available_end + _, available_end = await self._get_dataset_range(dataset) start = start or available_end - ONE_DAY end = end or available_end + if limit > 0: + self._log.warning( + f"Ignoring limit {limit} because its applied from the start (instead of the end).", + ) + self._log.info( f"Requesting {instrument_id} trade ticks: " - f"dataset={dataset}, start={start}, end={end}, limit={limit} ...", + f"dataset={dataset}, start={start}, end={end}", LogColor.BLUE, ) @@ -667,7 +700,6 @@ async def _request_trade_ticks( symbols=instrument_id.symbol.value, start=(start or available_end - ONE_DAY).value, end=(end or available_end).value, - limit=limit, ) trades = TradeTick.from_pyo3_list(pyo3_trades) @@ -687,18 +719,19 @@ async def _request_bars( end: pd.Timestamp | None = None, ) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(bar_type.instrument_id.venue) + _, available_end = await self._get_dataset_range(dataset) - # Check and cache dataset available end - available_end = self._available_ends.get(dataset) - if available_end is None: - _, available_end = await self._get_dataset_range(dataset) - self._available_ends[dataset] = available_end start = start or available_end - ONE_DAY end = end or available_end + if limit > 0: + self._log.warning( + f"Ignoring limit {limit} because its applied from the start (instead of the end).", + ) + self._log.info( f"Requesting {bar_type.instrument_id} 1 {bar_aggregation_to_str(bar_type.spec.aggregation)} bars: " - f"dataset={dataset}, start={start}, end={end}, limit={limit} ...", + f"dataset={dataset}, start={start}, end={end}", LogColor.BLUE, ) @@ -710,7 +743,6 @@ async def _request_bars( ), start=(start or available_end - ONE_DAY).value, end=(end or available_end).value, - limit=limit, ) bars = Bar.from_pyo3_list(pyo3_bars) From 97849bf23a2d37131eef83a63f2179bb9f1ee88b Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 08:50:04 +1100 Subject: [PATCH 05/42] Improve DatabentoDataClient config and startup --- .../live/databento/databento_subscriber.py | 1 + nautilus_trader/adapters/databento/config.py | 5 +++++ nautilus_trader/adapters/databento/data.py | 21 ++++++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/examples/live/databento/databento_subscriber.py b/examples/live/databento/databento_subscriber.py index f869ccb43eff..c3dddef2759d 100644 --- a/examples/live/databento/databento_subscriber.py +++ b/examples/live/databento/databento_subscriber.py @@ -80,6 +80,7 @@ http_gateway=None, instrument_provider=InstrumentProviderConfig(load_all=True), instrument_ids=instrument_ids, + parent_symbols={"GLBX.MDP3": {"ES.FUT", "ES.OPT"}}, ), }, timeout_connection=10.0, diff --git a/nautilus_trader/adapters/databento/config.py b/nautilus_trader/adapters/databento/config.py index 670426816189..64c7efd6a6c7 100644 --- a/nautilus_trader/adapters/databento/config.py +++ b/nautilus_trader/adapters/databento/config.py @@ -30,6 +30,10 @@ class DatabentoDataClientConfig(LiveDataClientConfig, frozen=True): The historical HTTP client gateway override. live_gateway : str, optional The live client gateway override. + parent_symbols : dict[str, set[str]], optional + The Databento parent symbols to subscribe to instrument definitions for on start. + This is a map of Databento dataset keys -> to a sequence of the parent symbols, + e.g. {'GLBX.MDP3', ['ES.FUT', 'ES.OPT']} (for all E-mini S&P 500 futures and options products). instrument_ids : list[InstrumentId], optional The instrument IDs to request instrument definitions for on start. timeout_initial_load : float, default 5.0 @@ -45,5 +49,6 @@ class DatabentoDataClientConfig(LiveDataClientConfig, frozen=True): http_gateway: str | None = None live_gateway: str | None = None instrument_ids: list[InstrumentId] | None = None + parent_symbols: dict[str, set[str]] | None = None timeout_initial_load: float | None = 5.0 mbo_subscriptions_delay: float | None = 3.0 diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index bd2c674daced..127170c6022b 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -113,6 +113,7 @@ def __init__( # Configuration self._live_api_key: str = config.api_key or http_client.key self._live_gateway: str | None = config.live_gateway + self._parent_symbols: dict[Dataset, set[str]] = defaultdict(set) self._instrument_ids: dict[Dataset, set[InstrumentId]] = defaultdict(set) self._timeout_initial_load: float | None = config.timeout_initial_load self._mbo_subscriptions_delay: float | None = config.mbo_subscriptions_delay @@ -131,9 +132,13 @@ def __init__( self._dataset_ranges: dict[Dataset, tuple[pd.Timestamp, pd.Timestamp]] = {} self._dataset_ranges_requested: set[Dataset] = set() + # Cache parent symbol index + for dataset, parent_symbols in (config.parent_symbols or {}).items(): + self._parent_symbols[dataset].update(set(parent_symbols)) + # Cache instrument index for instrument_id in config.instrument_ids or []: - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + dataset = self._loader.get_dataset_for_venue(instrument_id.venue) self._instrument_ids[dataset].add(instrument_id) # MBO/L3 subscription buffering @@ -360,6 +365,20 @@ async def _subscribe_instrument(self, instrument_id: InstrumentId) -> None: ) self._check_live_client_started(dataset, live_client) + async def _subscribe_parent_symbols( + self, + dataset: Dataset, + parent_symbols: set[str], + ) -> None: + live_client = self._get_live_client(dataset) + live_client.subscribe( + dataset=dataset, + stype_in=databento.SType.PARENT, + schema=databento.Schema.DEFINITION, + symbols=parent_symbols, + ) + self._check_live_client_started(dataset, live_client) + async def _subscribe_instrument_ids( self, dataset: Dataset, From 323a46085defc172357c5127bfc0aec6ae6accc6 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 09:04:08 +1100 Subject: [PATCH 06/42] Remove DatabentoDataLoader databento dependency --- nautilus_trader/adapters/databento/loaders.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index ce6a734d94d0..42df78dc2f6b 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -16,7 +16,7 @@ from os import PathLike from pathlib import Path -import databento +import databento_dbn import msgspec from nautilus_trader.adapters.databento.common import check_file_path @@ -31,6 +31,7 @@ from nautilus_trader.model.identifiers import InstrumentId from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.instruments import Instrument +from nautilus_trader.model.instruments import instruments_from_pyo3 class DatabentoDataLoader: @@ -188,35 +189,37 @@ def load_from_file_pyo3( raise RuntimeError("Loading files with mixed schemas not currently supported") match schema: - case databento.Schema.DEFINITION: - # TODO: pyo3 -> Cython conversion - return self._pyo3_loader.load_instruments(path) # type: ignore - case databento.Schema.MBO: + case databento_dbn.Schema.DEFINITION: + data = self._pyo3_loader.load_instruments(path) # type: ignore + if as_legacy_cython: + data = instruments_from_pyo3(data) + return data + case databento_dbn.Schema.MBO: data = self._pyo3_loader.load_order_book_deltas(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: data = OrderBookDelta.from_pyo3_list(data) return data - case databento.Schema.MBP_1 | databento.Schema.TBBO: + case databento_dbn.Schema.MBP_1 | databento_dbn.Schema.TBBO: data = self._pyo3_loader.load_quote_ticks(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: data = QuoteTick.from_pyo3_list(data) return data - case databento.Schema.MBP_10: + case databento_dbn.Schema.MBP_10: data = self._pyo3_loader.load_order_book_depth10(path) # type: ignore if as_legacy_cython: data = OrderBookDepth10.from_pyo3_list(data) return data - case databento.Schema.TRADES: + case databento_dbn.Schema.TRADES: data = self._pyo3_loader.load_trade_ticks(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: data = TradeTick.from_pyo3_list(data) return data case ( - databento.Schema.OHLCV_1S - | databento.Schema.OHLCV_1M - | databento.Schema.OHLCV_1H - | databento.Schema.OHLCV_1D - | databento.Schema.OHLCV_EOD + databento_dbn.Schema.OHLCV_1S + | databento_dbn.Schema.OHLCV_1M + | databento_dbn.Schema.OHLCV_1H + | databento_dbn.Schema.OHLCV_1D + | databento_dbn.Schema.OHLCV_EOD ): data = self._pyo3_loader.load_bars(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: From adc776e8eb72f7a41f620675778022828d6131c3 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 09:34:18 +1100 Subject: [PATCH 07/42] Use pyo3 DatabentoHistoricalClient exclusively --- .../live/databento/databento_subscriber.py | 2 ++ .../src/databento/python/historical.rs | 5 +++- nautilus_trader/adapters/databento/common.py | 15 +++------- nautilus_trader/adapters/databento/data.py | 8 +---- .../adapters/databento/factories.py | 20 +++++++------ .../adapters/databento/providers.py | 29 ++++++++----------- nautilus_trader/core/nautilus_pyo3.pyi | 2 ++ 7 files changed, 36 insertions(+), 45 deletions(-) diff --git a/examples/live/databento/databento_subscriber.py b/examples/live/databento/databento_subscriber.py index c3dddef2759d..b6f89e7bd6b1 100644 --- a/examples/live/databento/databento_subscriber.py +++ b/examples/live/databento/databento_subscriber.py @@ -26,6 +26,8 @@ from nautilus_trader.config.common import StrategyConfig from nautilus_trader.live.node import TradingNode from nautilus_trader.model.book import OrderBook + +# from nautilus_trader.model.data import BarType from nautilus_trader.model.data import OrderBookDeltas from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.data import TradeTick diff --git a/nautilus_core/adapters/src/databento/python/historical.rs b/nautilus_core/adapters/src/databento/python/historical.rs index 4c3b370069b8..526ec6ead0eb 100644 --- a/nautilus_core/adapters/src/databento/python/historical.rs +++ b/nautilus_core/adapters/src/databento/python/historical.rs @@ -50,6 +50,8 @@ pub struct DatabentoHistoricalClient { clock: &'static AtomicTime, inner: Arc>, publishers: Arc>, + #[pyo3(get)] + pub key: String, } #[pymethods] @@ -57,7 +59,7 @@ impl DatabentoHistoricalClient { #[new] pub fn py_new(key: String, publishers_path: &str) -> PyResult { let client = databento::HistoricalClient::builder() - .key(key) + .key(key.clone()) .map_err(to_pyvalue_err)? .build() .map_err(to_pyvalue_err)?; @@ -75,6 +77,7 @@ impl DatabentoHistoricalClient { clock: get_atomic_clock_realtime(), inner: Arc::new(Mutex::new(client)), publishers: Arc::new(publishers), + key, }) } diff --git a/nautilus_trader/adapters/databento/common.py b/nautilus_trader/adapters/databento/common.py index 0c7e839e86d8..f3152c92b8ba 100644 --- a/nautilus_trader/adapters/databento/common.py +++ b/nautilus_trader/adapters/databento/common.py @@ -15,10 +15,8 @@ from pathlib import Path -import databento import databento_dbn -import nautilus_trader from nautilus_trader.adapters.databento.types import DatabentoPublisher from nautilus_trader.core.correctness import PyCondition from nautilus_trader.model.data import BarType @@ -29,11 +27,6 @@ from nautilus_trader.model.identifiers import Venue -# Update Databento user-agent constant value with NautilusTrader version -if nautilus_trader.USER_AGENT not in databento.common.system.USER_AGENT: - databento.common.system.USER_AGENT += f" {nautilus_trader.USER_AGENT}" - - def check_file_path(path: Path) -> None: """ Check that the file at the given `path` exists and is not empty. @@ -126,13 +119,13 @@ def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> databento_dbn. match bar_type.spec.aggregation: case BarAggregation.SECOND: - return databento.Schema.OHLCV_1S + return databento_dbn.Schema.OHLCV_1S case BarAggregation.MINUTE: - return databento.Schema.OHLCV_1M + return databento_dbn.Schema.OHLCV_1M case BarAggregation.HOUR: - return databento.Schema.OHLCV_1H + return databento_dbn.Schema.OHLCV_1H case BarAggregation.DAY: - return databento.Schema.OHLCV_1D + return databento_dbn.Schema.OHLCV_1D case _: raise ValueError( f"Invalid bar type '{bar_type}'. " diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 127170c6022b..c3bbff05e674 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -17,7 +17,6 @@ from collections import defaultdict from collections.abc import Coroutine from functools import partial -from pathlib import Path from typing import Any import databento @@ -118,13 +117,8 @@ def __init__( self._timeout_initial_load: float | None = config.timeout_initial_load self._mbo_subscriptions_delay: float | None = config.mbo_subscriptions_delay - publishers_path = Path(__file__).resolve().parent / "publishers.json" - # Clients - self._http_client = nautilus_pyo3.DatabentoHistoricalClient( - key=http_client.key, - publishers_path=str(publishers_path.resolve()), - ) + self._http_client = http_client self._live_clients: dict[Dataset, databento.Live] = {} self._live_clients_mbo: dict[Dataset, databento.Live] = {} self._has_subscribed: dict[Dataset, bool] = {} diff --git a/nautilus_trader/adapters/databento/factories.py b/nautilus_trader/adapters/databento/factories.py index 479ecca6ccf3..5928e51dfc56 100644 --- a/nautilus_trader/adapters/databento/factories.py +++ b/nautilus_trader/adapters/databento/factories.py @@ -15,8 +15,7 @@ import asyncio from functools import lru_cache - -import databento +from pathlib import Path from nautilus_trader.adapters.databento.config import DatabentoDataClientConfig from nautilus_trader.adapters.databento.data import DatabentoDataClient @@ -27,17 +26,18 @@ from nautilus_trader.common.component import LiveClock from nautilus_trader.common.component import MessageBus from nautilus_trader.config.common import InstrumentProviderConfig +from nautilus_trader.core import nautilus_pyo3 from nautilus_trader.live.factories import LiveDataClientFactory -DATABENTO_HTTP_CLIENTS: dict[str, databento.Historical] = {} +DATABENTO_HTTP_CLIENTS: dict[str, nautilus_pyo3.DatabentoHistoricalClient] = {} @lru_cache(1) def get_cached_databento_http_client( key: str | None = None, gateway: str | None = None, -) -> databento.Historical: +) -> nautilus_pyo3.DatabentoHistoricalClient: """ Cache and return a Databento historical HTTP client with the given key and gateway. @@ -53,23 +53,25 @@ def get_cached_databento_http_client( Returns ------- - databento.Historical + nautilus_pyo3.DatabentoHistoricalClient """ global BINANCE_HTTP_CLIENTS key = key or get_env_key("DATABENTO_API_KEY") + publishers_path = str((Path(__file__).resolve().parent / "publishers.json").resolve()) + client_key: str = "|".join((key, gateway or "")) if client_key not in DATABENTO_HTTP_CLIENTS: - client = databento.Historical(key=key, gateway=gateway or databento.HistoricalGateway.BO1) + client = nautilus_pyo3.DatabentoHistoricalClient(key=key, publishers_path=publishers_path) DATABENTO_HTTP_CLIENTS[client_key] = client return DATABENTO_HTTP_CLIENTS[client_key] @lru_cache(1) def get_cached_databento_instrument_provider( - http_client: databento.Historical, + http_client: nautilus_pyo3.DatabentoHistoricalClient, clock: LiveClock, live_api_key: str | None = None, live_gateway: str | None = None, @@ -83,7 +85,7 @@ def get_cached_databento_instrument_provider( Parameters ---------- - http_client : databento.Historical + http_client : nautilus_pyo3.DatabentoHistoricalClient The client for the instrument provider. clock : LiveClock The clock for the instrument provider. @@ -150,7 +152,7 @@ def create( # type: ignore """ # Get HTTP client singleton - http_client: databento.Historical = get_cached_databento_http_client( + http_client = get_cached_databento_http_client( key=config.api_key, gateway=config.http_gateway, ) diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index a4028e40a0cf..40eff48036f6 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -17,15 +17,19 @@ import databento import pandas as pd +import pytz +from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.adapters.databento.parsing import parse_record_with_metadata from nautilus_trader.common.component import LiveClock from nautilus_trader.common.providers import InstrumentProvider from nautilus_trader.config import InstrumentProviderConfig +from nautilus_trader.core import nautilus_pyo3 from nautilus_trader.core.correctness import PyCondition from nautilus_trader.model.identifiers import InstrumentId from nautilus_trader.model.instruments import Instrument +from nautilus_trader.model.instruments import instruments_from_pyo3 class DatabentoInstrumentProvider(InstrumentProvider): @@ -34,7 +38,7 @@ class DatabentoInstrumentProvider(InstrumentProvider): Parameters ---------- - http_client : databento.Historical + http_client : nautilus_pyo3.DatabentoHistoricalClient The Databento historical HTTP client for the provider. clock : LiveClock The clock for the provider. @@ -52,7 +56,7 @@ class DatabentoInstrumentProvider(InstrumentProvider): def __init__( self, - http_client: databento.Historical, + http_client: nautilus_pyo3.DatabentoHistoricalClient, clock: LiveClock, live_api_key: str | None = None, live_gateway: str | None = None, @@ -202,24 +206,15 @@ async def get_range( """ dataset = self._check_all_datasets_equal(instrument_ids) - data = await self._http_client.timeseries.get_range_async( + + pyo3_instruments = await self._http_client.get_range_instruments( dataset=dataset, - schema=databento.Schema.DEFINITION, - start=start, - end=end, - symbols=[i.symbol.value for i in instrument_ids], - stype_in=databento.SType.RAW_SYMBOL, + symbols=ALL_SYMBOLS, + start=pd.Timestamp(start, tz=pytz.utc).value, + end=pd.Timestamp(end, tz=pytz.utc).value if end is not None else None, ) - instruments: list[Instrument] = [] - - for record in data: - instrument = parse_record_with_metadata( - record, - publishers=self._loader.publishers, - ts_init=self._clock.timestamp_ns(), - ) - instruments.append(instrument) + instruments = instruments_from_pyo3(pyo3_instruments) instruments = sorted(instruments, key=lambda x: x.ts_init) return instruments diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 4aaf53f740ba..86377361da9d 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1635,6 +1635,8 @@ class DatabentoHistoricalClient: publishers_path: str, ) -> None: ... + @property + def key(self) -> str: ... async def get_dataset_range(self, dataset: str) -> dict[str, str]: ... async def get_range_instruments( self, From 359020887473cc7279ffc02a6c291ada841cc51e Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 13:07:59 +1100 Subject: [PATCH 08/42] Remove databento_dbn common dependency --- nautilus_trader/adapters/databento/common.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/nautilus_trader/adapters/databento/common.py b/nautilus_trader/adapters/databento/common.py index f3152c92b8ba..3f92c7cff3fc 100644 --- a/nautilus_trader/adapters/databento/common.py +++ b/nautilus_trader/adapters/databento/common.py @@ -15,8 +15,6 @@ from pathlib import Path -import databento_dbn - from nautilus_trader.adapters.databento.types import DatabentoPublisher from nautilus_trader.core.correctness import PyCondition from nautilus_trader.model.data import BarType @@ -81,9 +79,9 @@ def nautilus_instrument_id_from_databento( return InstrumentId(Symbol(raw_symbol), Venue(publisher.venue)) -def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> databento_dbn.Schema: +def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> str: """ - Return the Databento bar aggregate schema for the given Nautilus `bar_type`. + Return the Databento bar aggregate schema string for the given Nautilus `bar_type`. Parameters ---------- @@ -92,7 +90,7 @@ def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> databento_dbn. Returns ------- - databento.Schema + str Raises ------ @@ -119,13 +117,13 @@ def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> databento_dbn. match bar_type.spec.aggregation: case BarAggregation.SECOND: - return databento_dbn.Schema.OHLCV_1S + return "ohlcv-1s" case BarAggregation.MINUTE: - return databento_dbn.Schema.OHLCV_1M + return "ohlcv-1m" case BarAggregation.HOUR: - return databento_dbn.Schema.OHLCV_1H + return "ohlcv-1h" case BarAggregation.DAY: - return databento_dbn.Schema.OHLCV_1D + return "ohlcv-1d" case _: raise ValueError( f"Invalid bar type '{bar_type}'. " From 895009214af2b0f2910a14479ff07c8e08ec8694 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 27 Jan 2024 17:49:05 +1100 Subject: [PATCH 09/42] Implement DatabentoLiveClient in Rust --- nautilus_core/Cargo.lock | 1 + nautilus_core/Cargo.toml | 1 + nautilus_core/adapters/Cargo.toml | 1 + .../adapters/src/databento/python/live.rs | 224 ++++++++++++++++++ .../adapters/src/databento/python/mod.rs | 1 + nautilus_core/common/Cargo.toml | 2 +- nautilus_core/pyo3/src/lib.rs | 5 +- nautilus_trader/adapters/databento/data.py | 196 +++++++-------- nautilus_trader/core/nautilus_pyo3.pyi | 26 +- 9 files changed, 359 insertions(+), 98 deletions(-) create mode 100644 nautilus_core/adapters/src/databento/python/live.rs diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index b505662248cb..73b7171ecaf7 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2424,6 +2424,7 @@ dependencies = [ "dbn", "indexmap 2.1.0", "itoa", + "log", "nautilus-common", "nautilus-core", "nautilus-model", diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 0c90abc1a69e..f2affd857851 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -29,6 +29,7 @@ futures = "0.3.30" indexmap = "2.1.0" itoa = "1.0.10" once_cell = "1.19.0" +log = { version = "0.4.20", features = ["std", "kv_unstable", "serde", "release_max_level_debug"] } pyo3 = { version = "0.20.2", features = ["rust_decimal"] } pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime", "tokio", "attributes"] } rand = "0.8.5" diff --git a/nautilus_core/adapters/Cargo.toml b/nautilus_core/adapters/Cargo.toml index f55ab1272258..e4ea251c3b9d 100644 --- a/nautilus_core/adapters/Cargo.toml +++ b/nautilus_core/adapters/Cargo.toml @@ -18,6 +18,7 @@ anyhow = { workspace = true } chrono = { workspace = true } indexmap = { workspace = true } itoa = { workspace = true } +log = { workspace = true } pyo3 = { workspace = true, optional = true } pyo3-asyncio = { workspace = true, optional = true } rand = { workspace = true } diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs new file mode 100644 index 000000000000..e4d3461c342a --- /dev/null +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -0,0 +1,224 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::fs; +use std::str::FromStr; +use std::sync::{Arc, OnceLock}; + +use anyhow::Result; +use databento::live::Subscription; +use dbn::{PitSymbolMap, RType, Record, SymbolIndex}; +use indexmap::IndexMap; +use log::error; +use nautilus_core::{ + python::to_pyvalue_err, + time::{get_atomic_clock_realtime, UnixNanos}, +}; +use nautilus_model::data::Data; +use nautilus_model::identifiers::instrument_id::InstrumentId; +use nautilus_model::identifiers::symbol::Symbol; +use nautilus_model::identifiers::venue::Venue; +use pyo3::prelude::*; +use tokio::sync::Mutex; +use ustr::Ustr; + +use crate::databento::parsing::parse_record; +use crate::databento::types::{DatabentoPublisher, PublisherId}; + +#[cfg_attr( + feature = "python", + pyclass(module = "nautilus_trader.core.nautilus_pyo3.databento") +)] +pub struct DatabentoLiveClient { + #[pyo3(get)] + pub key: String, + #[pyo3(get)] + pub dataset: String, + inner: OnceLock>>, + publishers: Arc>, +} + +impl DatabentoLiveClient { + async fn initialize_client(&self) -> Result { + databento::LiveClient::builder() + .key(&self.key)? + .dataset(&self.dataset) + .build() + .await + } + + fn get_inner_client(&self) -> Result>, databento::Error> { + if let Some(client) = self.inner.get() { + Ok(client.clone()) + } else { + // TODO: Improve the efficiency of this (shouldn't need a whole new runtime) + let runtime = tokio::runtime::Runtime::new().unwrap(); + let client = runtime.block_on(self.initialize_client())?; + let arc_client = Arc::new(Mutex::new(client)); + + // Ignore the result: In the current logic flow, the OnceLock should not be already set. + // However, this assumption might not hold in all contexts, especially in concurrent + // scenarios. + // TODO: Review and ensure thread safety and correct initialization logic. + let _ = self.inner.set(arc_client.clone()); + Ok(arc_client) + } + } +} + +#[pymethods] +impl DatabentoLiveClient { + #[new] + pub fn py_new(key: String, dataset: String, publishers_path: String) -> PyResult { + let file_content = fs::read_to_string(publishers_path)?; + let publishers_vec: Vec = + serde_json::from_str(&file_content).map_err(to_pyvalue_err)?; + let publishers = publishers_vec + .clone() + .into_iter() + .map(|p| (p.publisher_id, p)) + .collect::>(); + + Ok(Self { + key, + dataset, + inner: OnceLock::new(), + publishers: Arc::new(publishers), + }) + } + + #[pyo3(name = "subscribe")] + fn py_subscribe<'py>( + &self, + py: Python<'py>, + schema: String, + symbols: String, + stype_in: Option, + start: Option, + ) -> PyResult<&'py PyAny> { + println!("{:?}", start); // TODO: Debugging placeholder + // TODO: Properly map `start` + // let start = match start { + // Some(start) => Some( + // OffsetDateTime::from_unix_timestamp_nanos(start as i128).map_err(to_pyvalue_err)?, + // ), + // None => None, + // }; + + let stype_in = stype_in.unwrap_or("raw_symbol".to_string()); + let arc_client = self.get_inner_client().map_err(to_pyvalue_err)?; + + pyo3_asyncio::tokio::future_into_py(py, async move { + let mut client = arc_client.lock().await; + let subscription = Subscription::builder() + .symbols(symbols) + .schema(dbn::Schema::from_str(&schema).map_err(to_pyvalue_err)?) + .stype_in(dbn::SType::from_str(&stype_in).map_err(to_pyvalue_err)?) + // .start(start.expect(&format!("Invalid `start` was {:?}", start))) + .build(); + + client + .subscribe(&subscription) + .await + .map_err(to_pyvalue_err)?; + Ok(()) + }) + } + + #[pyo3(name = "start")] + fn py_start<'py>(&self, py: Python<'py>, callback: PyObject) -> PyResult<&'py PyAny> { + let arc_client = self.get_inner_client().map_err(to_pyvalue_err)?; + let publishers = self.publishers.clone(); + + pyo3_asyncio::tokio::future_into_py(py, async move { + let mut client = arc_client.lock().await; + let clock = get_atomic_clock_realtime(); + + let mut symbol_map = PitSymbolMap::new(); + while let Some(record) = client.next_record().await.map_err(to_pyvalue_err)? { + let rtype = record.rtype().expect("Invalid `rtype`"); + match rtype { + RType::SymbolMapping => { + symbol_map.on_record(record).unwrap_or_else(|_| { + panic!("Error updating `symbol_map` with {:?}", record) + }); + continue; + } + RType::Error => { + eprintln!("{:?}", record); // TODO: Just print stderr for now + error!("{:?}", record); + continue; + } + RType::System => { + eprintln!("{:?}", record); // TODO: Just print stderr for now + error!("{:?}", record); + continue; + } + _ => {} // Fall through + } + + let raw_symbol = symbol_map + .get_for_rec(&record) + .expect("Cannot resolve raw_symbol from `symbol_map`"); + + let symbol = Symbol { + value: Ustr::from(raw_symbol), + }; + + let publisher_id = record.publisher().unwrap() as PublisherId; + let venue_str = publishers.get(&publisher_id).unwrap().venue.as_str(); + + let venue = Venue { + value: Ustr::from(venue_str), + }; + + let instrument_id = InstrumentId::new(symbol, venue); + let ts_init = clock.get_time_ns(); + + let (data, _) = parse_record(&record, rtype, instrument_id, 2, Some(ts_init)) + .map_err(to_pyvalue_err)?; + + // TODO: Improve the efficiency of this constant GIL aquisition + Python::with_gil(|py| { + let data = match data { + Data::Delta(delta) => delta.into_py(py), + Data::Depth10(depth) => depth.into_py(py), + Data::Quote(quote) => quote.into_py(py), + Data::Trade(trade) => trade.into_py(py), + _ => panic!("Invalid data element, was {:?}", data), + }; + + match callback.call1(py, (data,)) { + Ok(_) => {} + Err(e) => eprintln!("{:?}", e), // Just print error for now + }; + }) + } + Ok(()) + }) + } + + // TODO: Close wants to take ownership which isn't possible? + #[pyo3(name = "close")] + fn py_close<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> { + // let arc_client = self.get_inner_client().map_err(to_pyvalue_err)?; + + pyo3_asyncio::tokio::future_into_py(py, async move { + // let client = arc_client.lock().await; + // client.close().await.map_err(to_pyvalue_err)?; + Ok(()) + }) + } +} diff --git a/nautilus_core/adapters/src/databento/python/mod.rs b/nautilus_core/adapters/src/databento/python/mod.rs index bfb24dcd1025..b24d96d0ae56 100644 --- a/nautilus_core/adapters/src/databento/python/mod.rs +++ b/nautilus_core/adapters/src/databento/python/mod.rs @@ -14,5 +14,6 @@ // ------------------------------------------------------------------------------------------------- pub mod historical; +pub mod live; pub mod loader; pub mod parsing; diff --git a/nautilus_core/common/Cargo.toml b/nautilus_core/common/Cargo.toml index 3428031e235a..56635a837493 100644 --- a/nautilus_core/common/Cargo.toml +++ b/nautilus_core/common/Cargo.toml @@ -16,6 +16,7 @@ nautilus-model = { path = "../model", features = ["stubs"]} anyhow = { workspace = true } chrono = { workspace = true } indexmap = { workspace = true } +log = { workspace = true } pyo3 = { workspace = true, optional = true } redis = { workspace = true, optional = true } serde = { workspace = true } @@ -23,7 +24,6 @@ serde_json = { workspace = true } strum = { workspace = true } ustr = { workspace = true } tracing = { workspace = true } -log = { version = "0.4.20", features = ["std", "kv_unstable", "serde", "release_max_level_debug"] } sysinfo = "0.30.5" # Disable default feature "tracing-log" since it interferes with custom logging tracing-subscriber = { version = "0.3.18", default-features = false, features = ["smallvec", "fmt", "ansi", "std", "env-filter"] } diff --git a/nautilus_core/pyo3/src/lib.rs b/nautilus_core/pyo3/src/lib.rs index e188ee1548af..ea28254a9125 100644 --- a/nautilus_core/pyo3/src/lib.rs +++ b/nautilus_core/pyo3/src/lib.rs @@ -13,7 +13,9 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use nautilus_adapters::databento::{loader, python::historical, python::parsing, types}; +use nautilus_adapters::databento::{ + loader, python::historical, python::live, python::parsing, types, +}; use pyo3::{ prelude::*, types::{PyDict, PyString}, @@ -27,6 +29,7 @@ use pyo3::{ pub fn databento(_: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(parsing::py_parse_equity, m)?)?; m.add_function(wrap_pyfunction!(parsing::py_parse_futures_contract, m)?)?; diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index c3bbff05e674..f94d629a67d3 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -16,23 +16,20 @@ import asyncio from collections import defaultdict from collections.abc import Coroutine -from functools import partial +from pathlib import Path from typing import Any -import databento +import databento_dbn import pandas as pd import pytz from nautilus_trader.adapters.databento.common import databento_schema_from_nautilus_bar_type -from nautilus_trader.adapters.databento.common import nautilus_instrument_id_from_databento from nautilus_trader.adapters.databento.config import DatabentoDataClientConfig from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID from nautilus_trader.adapters.databento.constants import ONE_DAY from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader -from nautilus_trader.adapters.databento.parsing import parse_record from nautilus_trader.adapters.databento.providers import DatabentoInstrumentProvider -from nautilus_trader.adapters.databento.types import DatabentoPublisher from nautilus_trader.adapters.databento.types import Dataset from nautilus_trader.cache.cache import Cache from nautilus_trader.common.component import LiveClock @@ -47,6 +44,7 @@ from nautilus_trader.model.data import DataType from nautilus_trader.model.data import OrderBookDelta from nautilus_trader.model.data import OrderBookDeltas +from nautilus_trader.model.data import OrderBookDepth10 from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.data import TradeTick from nautilus_trader.model.enums import BookType @@ -64,7 +62,7 @@ class DatabentoDataClient(LiveMarketDataClient): ---------- loop : asyncio.AbstractEventLoop The event loop for the client. - http_client : databento.Historical + http_client : nautilus_pyo3.DatabentoHistoricalClient The Databento historical HTTP client. msgbus : MessageBus The message bus for the client. @@ -86,7 +84,7 @@ class DatabentoDataClient(LiveMarketDataClient): def __init__( self, loop: asyncio.AbstractEventLoop, - http_client: databento.Historical, + http_client: nautilus_pyo3.DatabentoHistoricalClient, msgbus: MessageBus, cache: Cache, clock: LiveClock, @@ -109,6 +107,9 @@ def __init__( config=config, ) + # TODO: Single source this + self._publishers_path = str((Path(__file__).resolve().parent / "publishers.json").resolve()) + # Configuration self._live_api_key: str = config.api_key or http_client.key self._live_gateway: str | None = config.live_gateway @@ -119,8 +120,8 @@ def __init__( # Clients self._http_client = http_client - self._live_clients: dict[Dataset, databento.Live] = {} - self._live_clients_mbo: dict[Dataset, databento.Live] = {} + self._live_clients: dict[Dataset, nautilus_pyo3.DatabentoLiveClient] = {} + self._live_clients_mbo: dict[Dataset, nautilus_pyo3.DatabentoLiveClient] = {} self._has_subscribed: dict[Dataset, bool] = {} self._loader = loader or DatabentoDataLoader() self._dataset_ranges: dict[Dataset, tuple[pd.Timestamp, pd.Timestamp]] = {} @@ -142,6 +143,7 @@ def __init__( self._buffered_deltas: dict[InstrumentId, list[OrderBookDelta]] = defaultdict(list) # Tasks + self._live_client_tasks: set[asyncio.Task] = set() self._update_dataset_ranges_interval_seconds: int = 60 * 60 # Once per hour (hard coded) self._update_dataset_ranges_task: asyncio.Task | None = None @@ -173,15 +175,16 @@ async def _connect(self) -> None: async def _disconnect(self) -> None: coros: list[Coroutine] = [] - for dataset, client in self._live_clients.items(): - self._log.info(f"Stopping {dataset} live feed...", LogColor.BLUE) - coro = client.wait_for_close(timeout=2.0) - coros.append(coro) + # TODO: When closing live clients sorted + # for dataset, client in self._live_clients.items(): + # self._log.info(f"Stopping {dataset} live feed...", LogColor.BLUE) + # coro = client.wait_for_close(timeout=2.0) + # coros.append(coro) - for dataset, client in self._live_clients_mbo.items(): - self._log.info(f"Stopping {dataset} MBO/L3 live feed...", LogColor.BLUE) - coro = client.wait_for_close(timeout=2.0) - coros.append(coro) + # for dataset, client in self._live_clients_mbo.items(): + # self._log.info(f"Stopping {dataset} MBO/L3 live feed...", LogColor.BLUE) + # coro = client.wait_for_close(timeout=2.0) + # coros.append(coro) if self._buffer_mbo_subscriptions_task: self._log.debug("Canceling `buffer_mbo_subscriptions` task...") @@ -232,38 +235,51 @@ async def _buffer_mbo_subscriptions(self) -> None: except asyncio.CancelledError: self._log.debug("Canceled `buffer_mbo_subscriptions` task.") - def _get_live_client(self, dataset: Dataset) -> databento.Live: + def _get_live_client(self, dataset: Dataset) -> nautilus_pyo3.DatabentoLiveClient: # Retrieve or initialize the 'general' live client for the specified dataset live_client = self._live_clients.get(dataset) if live_client is None: - live_client = databento.Live(key=self._live_api_key, gateway=self._live_gateway) + live_client = nautilus_pyo3.DatabentoLiveClient( + key=self._live_api_key, + dataset=dataset, + publishers_path=self._publishers_path, + ) # Wrap the callback with partial to include the dataset - callback_with_dataset = partial(self._handle_record, live_client.symbology_map) - live_client.add_callback(callback_with_dataset) + # callback_with_dataset = partial(self._handle_record, live_client.symbology_map) + # live_client.add_callback(callback_with_dataset) self._live_clients[dataset] = live_client return live_client - def _get_live_client_mbo(self, dataset: Dataset) -> databento.Live: + def _get_live_client_mbo(self, dataset: Dataset) -> nautilus_pyo3.DatabentoLiveClient: # Retrieve or initialize the 'MBO/L3' live client for the specified dataset live_client = self._live_clients_mbo.get(dataset) if live_client is None: - live_client = databento.Live(key=self._live_api_key, gateway=self._live_gateway) + live_client = nautilus_pyo3.DatabentoLiveClient( + key=self._live_api_key, + dataset=dataset, + publishers_path=self._publishers_path, + ) # Wrap the callback with partial to include the dataset - callback_with_dataset = partial(self._handle_record, live_client.symbology_map) - live_client.add_callback(callback_with_dataset) + # callback_with_dataset = partial(self._handle_record, live_client.symbology_map) + # live_client.add_callback(callback_with_dataset) self._live_clients_mbo[dataset] = live_client return live_client - def _check_live_client_started(self, dataset: Dataset, live_client: databento.Live) -> None: + def _check_live_client_started( + self, + dataset: Dataset, + live_client: nautilus_pyo3.DatabentoLiveClient, + ) -> None: if not self._has_subscribed.get(dataset): self._log.debug(f"Starting {dataset} live client...", LogColor.MAGENTA) - live_client.start() + task = self._loop.create_task(live_client.start(callback=self._handle_record)) + self._live_client_tasks.add(task) self._has_subscribed[dataset] = True self._log.info(f"Started {dataset} live feed.", LogColor.BLUE) @@ -352,10 +368,9 @@ async def _subscribe_instruments(self) -> None: async def _subscribe_instrument(self, instrument_id: InstrumentId) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, - schema=databento.Schema.DEFINITION, - symbols=[instrument_id.symbol.value], + await live_client.subscribe( + schema="definition", + symbols=instrument_id.symbol.value, ) self._check_live_client_started(dataset, live_client) @@ -365,11 +380,10 @@ async def _subscribe_parent_symbols( parent_symbols: set[str], ) -> None: live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, - stype_in=databento.SType.PARENT, - schema=databento.Schema.DEFINITION, - symbols=parent_symbols, + await live_client.subscribe( + schema="definition", + symbols=",".join(sorted(parent_symbols)), + stype_in="parent", ) self._check_live_client_started(dataset, live_client) @@ -379,10 +393,9 @@ async def _subscribe_instrument_ids( instrument_ids: list[InstrumentId], ) -> None: live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, - schema=databento.Schema.DEFINITION, - symbols=[i.symbol.value for i in instrument_ids], + await live_client.subscribe( + schema="definition", + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), ) self._check_live_client_started(dataset, live_client) @@ -444,13 +457,13 @@ async def _subscribe_order_book_deltas_batch( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_ids[0].venue) live_client = self._get_live_client_mbo(dataset) - live_client.subscribe( - dataset=dataset, - schema=databento.Schema.MBO, - symbols=[i.symbol.value for i in instrument_ids], + await live_client.subscribe( + schema="mbo", + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now ) - live_client.start() + task = self._loop.create_task(live_client.start(self._handle_record)) + self._live_client_tasks.add(task) async def _subscribe_order_book_snapshots( self, @@ -463,9 +476,9 @@ async def _subscribe_order_book_snapshots( match depth: case 1: - schema = databento.Schema.MBP_1 + schema = "mbp-1" case 10: - schema = databento.Schema.MBP_10 + schema = "mbp-10" case _: self._log.error( f"Cannot subscribe for order book snapshots of depth {depth}, use either 1 or 10.", @@ -474,10 +487,9 @@ async def _subscribe_order_book_snapshots( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, + await live_client.subscribe( schema=schema, - symbols=[instrument_id.symbol.value], + symbols=",".join(sorted([instrument_id.symbol.value])), ) self._check_live_client_started(dataset, live_client) @@ -486,10 +498,9 @@ async def _subscribe_quote_ticks(self, instrument_id: InstrumentId) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, - schema=databento.Schema.MBP_1, - symbols=[instrument_id.symbol.value], + await live_client.subscribe( + schema="mbp-1", + symbols=",".join(sorted([instrument_id.symbol.value])), ) self._check_live_client_started(dataset, live_client) @@ -501,10 +512,9 @@ async def _subscribe_trade_ticks(self, instrument_id: InstrumentId) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, - schema=databento.Schema.TRADES, - symbols=[instrument_id.symbol.value], + await live_client.subscribe( + schema="trades", + symbols=instrument_id.symbol.value, ) self._check_live_client_started(dataset, live_client) @@ -518,10 +528,9 @@ async def _subscribe_bars(self, bar_type: BarType) -> None: return live_client = self._get_live_client(dataset) - live_client.subscribe( - dataset=dataset, + await live_client.subscribe( schema=schema, - symbols=[bar_type.instrument_id.symbol.value], + symbols=bar_type.instrument_id.symbol.value, ) self._check_live_client_started(dataset, live_client) @@ -769,48 +778,45 @@ async def _request_bars( def _handle_record( self, - instrument_map: dict[int, str | int], - record: databento.DBNRecord, + pyo3_data: Any, # TODO: Define `Data` base class ) -> None: # self._log.debug(f"Received {record}", LogColor.MAGENTA) - if isinstance(record, databento.ErrorMsg): - self._log.error(f"ErrorMsg: {record.err}") - return - elif isinstance(record, databento.SystemMsg): - self._log.info(f"SystemMsg: {record.msg}") - return - elif isinstance(record, databento.SymbolMappingMsg): - self._log.debug(f"SymbolMappingMsg: {record}") - return - try: - raw_symbol = instrument_map.get(record.instrument_id) - if raw_symbol is None: - raise ValueError(f"Cannot resolve instrument_id {record.instrument_id}") - - publisher: DatabentoPublisher = self._loader.publishers[record.publisher_id] - instrument_id: InstrumentId = nautilus_instrument_id_from_databento( - raw_symbol=str(raw_symbol), - publisher=publisher, - ) - data = parse_record(record, instrument_id, ts_init=self._clock.timestamp_ns()) - except ValueError as e: - self._log.error(f"{e!r}") - return + # TODO: Handle inside client + + # if isinstance(record, databento.ErrorMsg): + # self._log.error(f"ErrorMsg: {record.err}") + # return + # elif isinstance(record, databento.SystemMsg): + # self._log.info(f"SystemMsg: {record.msg}") + # return + # elif isinstance(record, databento.SymbolMappingMsg): + # self._log.debug(f"SymbolMappingMsg: {record}") + # return + + instrument_id = InstrumentId.from_str(pyo3_data.instrument_id.value) - if isinstance(data, OrderBookDelta): - if databento.RecordFlags.F_LAST not in databento.RecordFlags(data.flags): - buffer = self._buffered_deltas[data.instrument_id] + if isinstance(pyo3_data, nautilus_pyo3.OrderBookDelta): + data = OrderBookDelta.from_pyo3_list([pyo3_data]) # TODO: Implement single function + if databento_dbn.RecordFlags.F_LAST not in databento_dbn.RecordFlags(data.flags): + buffer = self._buffered_deltas[instrument_id] buffer.append(data) return # We can rely on the F_LAST flag for an MBO feed else: - buffer = self._buffered_deltas[data.instrument_id] + buffer = self._buffered_deltas[instrument_id] buffer.append(data) data = OrderBookDeltas(instrument_id, deltas=buffer.copy()) buffer.clear() - - if isinstance(data, tuple): - self._handle_data(data[0]) - self._handle_data(data[1]) + elif isinstance(pyo3_data, nautilus_pyo3.OrderBookDepth10): + data = OrderBookDepth10.from_pyo3_list([pyo3_data]) + elif isinstance(pyo3_data, nautilus_pyo3.QuoteTick): + data = QuoteTick.from_pyo3_list([pyo3_data]) + elif isinstance(pyo3_data, nautilus_pyo3.TradeTick): + data = TradeTick.from_pyo3_list([pyo3_data]) + elif isinstance(pyo3_data, nautilus_pyo3.Bar): + data = Bar.from_pyo3_list([pyo3_data]) else: - self._handle_data(data) + self._log.error(f"Unimplemented data type in handler: {pyo3_data}.") + return + + self._handle_data(data) diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 86377361da9d..981d80c0ea1d 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1627,7 +1627,6 @@ class DatabentoDataLoader: path: PathLike[str] | str, ) -> None: ... - class DatabentoHistoricalClient: def __init__( self, @@ -1671,3 +1670,28 @@ class DatabentoHistoricalClient: end: int | None = None, limit: int | None = None, ) -> list[Bar]: ... + +class DatabentoLiveClient: + def __init__( + self, + key: str, + dataset: str, + publishers_path: str, + ) -> None: ... + + @property + def key(self) -> str: ... + @property + def dataset(self) -> str: ... + async def subscribe( + self, + schema: str, + symbols: str, + stype_in: str | None = None, + start: int | None = None, + ) -> dict[str, str]: ... + async def start( + self, + callback: Callable, + ) -> dict[str, str]: ... + async def close(self) -> None: ... From 7e3ca7ff08b9471a84971d8d5a3f8473070aa867 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 07:30:04 +1100 Subject: [PATCH 10/42] Upgrade serde --- nautilus_core/Cargo.lock | 12 ++++++------ nautilus_core/Cargo.toml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 73b7171ecaf7..8bd2e496d005 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -3908,18 +3908,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -3928,9 +3928,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "4d1bd37ce2324cf3bf85e5a25f96eb4baf0d5aa6eba43e7ae8958870c4ec48ed" dependencies = [ "itoa", "ryu", diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index f2affd857851..8d965f11231e 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -37,8 +37,8 @@ redis = { version = "0.24.0", features = ["tokio-comp", "tls-rustls", "tokio-rus rmp-serde = "1.1.2" rust_decimal = "1.33.1" rust_decimal_macros = "1.33.1" -serde = { version = "1.0.195", features = ["derive"] } -serde_json = "1.0.111" +serde = { version = "1.0.196", features = ["derive"] } +serde_json = "1.0.112" strum = { version = "0.25.0", features = ["derive"] } thiserror = "1.0.56" thousands = "0.2.0" From 71d3a91e26d9248a853ce4792dceae77b291cdd8 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 09:16:39 +1100 Subject: [PATCH 11/42] Refine Databento clients --- .../adapters/src/databento/common.rs | 3 + .../adapters/src/databento/python/live.rs | 70 +++++++++---------- nautilus_core/model/src/identifiers/symbol.rs | 6 ++ nautilus_core/model/src/identifiers/venue.rs | 6 ++ nautilus_core/pyo3/src/lib.rs | 9 ++- .../adapters/databento/constants.py | 5 +- nautilus_trader/adapters/databento/data.py | 68 ++++++++---------- .../adapters/databento/factories.py | 9 +-- nautilus_trader/adapters/databento/loaders.py | 7 +- 9 files changed, 93 insertions(+), 90 deletions(-) diff --git a/nautilus_core/adapters/src/databento/common.rs b/nautilus_core/adapters/src/databento/common.rs index 15f45472fd84..a866859859ef 100644 --- a/nautilus_core/adapters/src/databento/common.rs +++ b/nautilus_core/adapters/src/databento/common.rs @@ -22,6 +22,9 @@ use ustr::Ustr; use super::types::DatabentoPublisher; +pub const DATABENTO: &str = "DATABENTO"; +pub const ALL_SYMBOLS: &str = "ALL_SYMBOLS"; + #[must_use] pub fn nautilus_instrument_id_from_databento( raw_symbol: Ustr, diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index e4d3461c342a..8b02401d8ddb 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -22,6 +22,7 @@ use databento::live::Subscription; use dbn::{PitSymbolMap, RType, Record, SymbolIndex}; use indexmap::IndexMap; use log::error; +use nautilus_core::python::to_pyruntime_err; use nautilus_core::{ python::to_pyvalue_err, time::{get_atomic_clock_realtime, UnixNanos}, @@ -31,8 +32,8 @@ use nautilus_model::identifiers::instrument_id::InstrumentId; use nautilus_model::identifiers::symbol::Symbol; use nautilus_model::identifiers::venue::Venue; use pyo3::prelude::*; +use time::OffsetDateTime; use tokio::sync::Mutex; -use ustr::Ustr; use crate::databento::parsing::parse_record; use crate::databento::types::{DatabentoPublisher, PublisherId}; @@ -47,6 +48,7 @@ pub struct DatabentoLiveClient { #[pyo3(get)] pub dataset: String, inner: OnceLock>>, + runtime: tokio::runtime::Runtime, publishers: Arc>, } @@ -63,15 +65,8 @@ impl DatabentoLiveClient { if let Some(client) = self.inner.get() { Ok(client.clone()) } else { - // TODO: Improve the efficiency of this (shouldn't need a whole new runtime) - let runtime = tokio::runtime::Runtime::new().unwrap(); - let client = runtime.block_on(self.initialize_client())?; + let client = self.runtime.block_on(self.initialize_client())?; let arc_client = Arc::new(Mutex::new(client)); - - // Ignore the result: In the current logic flow, the OnceLock should not be already set. - // However, this assumption might not hold in all contexts, especially in concurrent - // scenarios. - // TODO: Review and ensure thread safety and correct initialization logic. let _ = self.inner.set(arc_client.clone()); Ok(arc_client) } @@ -95,6 +90,7 @@ impl DatabentoLiveClient { key, dataset, inner: OnceLock::new(), + runtime: tokio::runtime::Runtime::new()?, publishers: Arc::new(publishers), }) } @@ -108,26 +104,30 @@ impl DatabentoLiveClient { stype_in: Option, start: Option, ) -> PyResult<&'py PyAny> { - println!("{:?}", start); // TODO: Debugging placeholder - // TODO: Properly map `start` - // let start = match start { - // Some(start) => Some( - // OffsetDateTime::from_unix_timestamp_nanos(start as i128).map_err(to_pyvalue_err)?, - // ), - // None => None, - // }; - let stype_in = stype_in.unwrap_or("raw_symbol".to_string()); - let arc_client = self.get_inner_client().map_err(to_pyvalue_err)?; + let arc_client = self.get_inner_client().map_err(to_pyruntime_err)?; pyo3_asyncio::tokio::future_into_py(py, async move { let mut client = arc_client.lock().await; - let subscription = Subscription::builder() - .symbols(symbols) - .schema(dbn::Schema::from_str(&schema).map_err(to_pyvalue_err)?) - .stype_in(dbn::SType::from_str(&stype_in).map_err(to_pyvalue_err)?) - // .start(start.expect(&format!("Invalid `start` was {:?}", start))) - .build(); + + // TODO: This can be tidied up, conditionally calling `if let Some(start)` on + // the builder was proving troublesome. + let subscription = match start { + Some(start) => Subscription::builder() + .symbols(symbols) + .schema(dbn::Schema::from_str(&schema).map_err(to_pyvalue_err)?) + .stype_in(dbn::SType::from_str(&stype_in).map_err(to_pyvalue_err)?) + .start( + OffsetDateTime::from_unix_timestamp_nanos(start as i128) + .map_err(to_pyvalue_err)?, + ) + .build(), + None => Subscription::builder() + .symbols(symbols) + .schema(dbn::Schema::from_str(&schema).map_err(to_pyvalue_err)?) + .stype_in(dbn::SType::from_str(&stype_in).map_err(to_pyvalue_err)?) + .build(), + }; client .subscribe(&subscription) @@ -139,14 +139,14 @@ impl DatabentoLiveClient { #[pyo3(name = "start")] fn py_start<'py>(&self, py: Python<'py>, callback: PyObject) -> PyResult<&'py PyAny> { - let arc_client = self.get_inner_client().map_err(to_pyvalue_err)?; + let arc_client = self.get_inner_client().map_err(to_pyruntime_err)?; let publishers = self.publishers.clone(); pyo3_asyncio::tokio::future_into_py(py, async move { - let mut client = arc_client.lock().await; let clock = get_atomic_clock_realtime(); - + let mut client = arc_client.lock().await; let mut symbol_map = PitSymbolMap::new(); + while let Some(record) = client.next_record().await.map_err(to_pyvalue_err)? { let rtype = record.rtype().expect("Invalid `rtype`"); match rtype { @@ -173,16 +173,10 @@ impl DatabentoLiveClient { .get_for_rec(&record) .expect("Cannot resolve raw_symbol from `symbol_map`"); - let symbol = Symbol { - value: Ustr::from(raw_symbol), - }; - + let symbol = Symbol::from_str_unchecked(raw_symbol); let publisher_id = record.publisher().unwrap() as PublisherId; let venue_str = publishers.get(&publisher_id).unwrap().venue.as_str(); - - let venue = Venue { - value: Ustr::from(venue_str), - }; + let venue = Venue::from_str_unchecked(venue_str); let instrument_id = InstrumentId::new(symbol, venue); let ts_init = clock.get_time_ns(); @@ -202,7 +196,7 @@ impl DatabentoLiveClient { match callback.call1(py, (data,)) { Ok(_) => {} - Err(e) => eprintln!("{:?}", e), // Just print error for now + Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now }; }) } @@ -216,7 +210,7 @@ impl DatabentoLiveClient { // let arc_client = self.get_inner_client().map_err(to_pyvalue_err)?; pyo3_asyncio::tokio::future_into_py(py, async move { - // let client = arc_client.lock().await; + // let client = arc_client.lock_owned().await; // client.close().await.map_err(to_pyvalue_err)?; Ok(()) }) diff --git a/nautilus_core/model/src/identifiers/symbol.rs b/nautilus_core/model/src/identifiers/symbol.rs index 2cc0f514a5e8..d6a09ca2cb08 100644 --- a/nautilus_core/model/src/identifiers/symbol.rs +++ b/nautilus_core/model/src/identifiers/symbol.rs @@ -42,6 +42,12 @@ impl Symbol { value: Ustr::from(s), }) } + + pub fn from_str_unchecked(s: &str) -> Self { + Self { + value: Ustr::from(s), + } + } } impl Default for Symbol { diff --git a/nautilus_core/model/src/identifiers/venue.rs b/nautilus_core/model/src/identifiers/venue.rs index 3db92c0f756a..cde3eaca7f78 100644 --- a/nautilus_core/model/src/identifiers/venue.rs +++ b/nautilus_core/model/src/identifiers/venue.rs @@ -45,6 +45,12 @@ impl Venue { }) } + pub fn from_str_unchecked(s: &str) -> Self { + Self { + value: Ustr::from(s), + } + } + #[must_use] pub fn synthetic() -> Self { // SAFETY: Unwrap safe as using known synthetic venue constant diff --git a/nautilus_core/pyo3/src/lib.rs b/nautilus_core/pyo3/src/lib.rs index ea28254a9125..44fcafc5de7e 100644 --- a/nautilus_core/pyo3/src/lib.rs +++ b/nautilus_core/pyo3/src/lib.rs @@ -14,7 +14,12 @@ // ------------------------------------------------------------------------------------------------- use nautilus_adapters::databento::{ - loader, python::historical, python::live, python::parsing, types, + common::{ALL_SYMBOLS, DATABENTO}, + loader, + python::historical, + python::live, + python::parsing, + types, }; use pyo3::{ prelude::*, @@ -27,6 +32,8 @@ use pyo3::{ /// Loaded as nautilus_pyo3.databento #[pymodule] pub fn databento(_: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add(stringify!(DATABENTO), DATABENTO)?; + m.add(stringify!(ALL_SYMBOLS), ALL_SYMBOLS)?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/nautilus_trader/adapters/databento/constants.py b/nautilus_trader/adapters/databento/constants.py index 08869b9c9806..b99f6e0e2a42 100644 --- a/nautilus_trader/adapters/databento/constants.py +++ b/nautilus_trader/adapters/databento/constants.py @@ -13,10 +13,9 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- +from pathlib import Path from typing import Final -import pandas as pd - from nautilus_trader.model.identifiers import ClientId @@ -25,4 +24,4 @@ ALL_SYMBOLS: Final[str] = "ALL_SYMBOLS" -ONE_DAY: Final[pd.Timedelta] = pd.Timedelta(days=1) +PUBLISHERS_PATH: Final[Path] = (Path(__file__).resolve().parent / "publishers.json").resolve() diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index f94d629a67d3..d530fbd8ca67 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -16,7 +16,6 @@ import asyncio from collections import defaultdict from collections.abc import Coroutine -from pathlib import Path from typing import Any import databento_dbn @@ -27,7 +26,7 @@ from nautilus_trader.adapters.databento.config import DatabentoDataClientConfig from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID -from nautilus_trader.adapters.databento.constants import ONE_DAY +from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.adapters.databento.providers import DatabentoInstrumentProvider from nautilus_trader.adapters.databento.types import Dataset @@ -107,9 +106,6 @@ def __init__( config=config, ) - # TODO: Single source this - self._publishers_path = str((Path(__file__).resolve().parent / "publishers.json").resolve()) - # Configuration self._live_api_key: str = config.api_key or http_client.key self._live_gateway: str | None = config.live_gateway @@ -143,7 +139,7 @@ def __init__( self._buffered_deltas: dict[InstrumentId, list[OrderBookDelta]] = defaultdict(list) # Tasks - self._live_client_tasks: set[asyncio.Task] = set() + self._live_client_futures: set[asyncio.Future] = set() self._update_dataset_ranges_interval_seconds: int = 60 * 60 # Once per hour (hard coded) self._update_dataset_ranges_task: asyncio.Task | None = None @@ -174,18 +170,6 @@ async def _connect(self) -> None: self._update_dataset_ranges_task = self.create_task(self._update_dataset_ranges()) async def _disconnect(self) -> None: - coros: list[Coroutine] = [] - # TODO: When closing live clients sorted - # for dataset, client in self._live_clients.items(): - # self._log.info(f"Stopping {dataset} live feed...", LogColor.BLUE) - # coro = client.wait_for_close(timeout=2.0) - # coros.append(coro) - - # for dataset, client in self._live_clients_mbo.items(): - # self._log.info(f"Stopping {dataset} MBO/L3 live feed...", LogColor.BLUE) - # coro = client.wait_for_close(timeout=2.0) - # coros.append(coro) - if self._buffer_mbo_subscriptions_task: self._log.debug("Canceling `buffer_mbo_subscriptions` task...") self._buffer_mbo_subscriptions_task.cancel() @@ -197,7 +181,19 @@ async def _disconnect(self) -> None: self._update_dataset_ranges_task.cancel() self._update_dataset_ranges_task = None - await asyncio.gather(*coros) + # Close all live clients + futures: list[asyncio.Future] = [] + for dataset, live_client in self._live_clients.items(): + self._log.info(f"Stopping {dataset} live feed...", LogColor.BLUE) + future = asyncio.ensure_future(live_client.close()) + futures.append(future) + + for dataset, live_client in self._live_clients_mbo.items(): + self._log.info(f"Stopping {dataset} MBO/L3 live feed...", LogColor.BLUE) + future = asyncio.ensure_future(live_client.close()) + futures.append(future) + + await asyncio.gather(*futures) async def _update_dataset_ranges(self) -> None: while True: @@ -243,12 +239,8 @@ def _get_live_client(self, dataset: Dataset) -> nautilus_pyo3.DatabentoLiveClien live_client = nautilus_pyo3.DatabentoLiveClient( key=self._live_api_key, dataset=dataset, - publishers_path=self._publishers_path, + publishers_path=str(PUBLISHERS_PATH), ) - - # Wrap the callback with partial to include the dataset - # callback_with_dataset = partial(self._handle_record, live_client.symbology_map) - # live_client.add_callback(callback_with_dataset) self._live_clients[dataset] = live_client return live_client @@ -261,12 +253,8 @@ def _get_live_client_mbo(self, dataset: Dataset) -> nautilus_pyo3.DatabentoLiveC live_client = nautilus_pyo3.DatabentoLiveClient( key=self._live_api_key, dataset=dataset, - publishers_path=self._publishers_path, + publishers_path=str(PUBLISHERS_PATH), ) - - # Wrap the callback with partial to include the dataset - # callback_with_dataset = partial(self._handle_record, live_client.symbology_map) - # live_client.add_callback(callback_with_dataset) self._live_clients_mbo[dataset] = live_client return live_client @@ -278,8 +266,8 @@ def _check_live_client_started( ) -> None: if not self._has_subscribed.get(dataset): self._log.debug(f"Starting {dataset} live client...", LogColor.MAGENTA) - task = self._loop.create_task(live_client.start(callback=self._handle_record)) - self._live_client_tasks.add(task) + future = asyncio.ensure_future(live_client.start(callback=self._handle_record)) + self._live_client_futures.add(future) self._has_subscribed[dataset] = True self._log.info(f"Started {dataset} live feed.", LogColor.BLUE) @@ -462,8 +450,8 @@ async def _subscribe_order_book_deltas_batch( symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now ) - task = self._loop.create_task(live_client.start(self._handle_record)) - self._live_client_tasks.add(task) + future = asyncio.ensure_future(live_client.start(self._handle_record)) + self._live_client_futures.add(future) async def _subscribe_order_book_snapshots( self, @@ -595,7 +583,7 @@ async def _request_instrument( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) _, available_end = await self._get_dataset_range(dataset) - start = start or available_end - ONE_DAY * 2 + start = start or available_end - pd.Timedelta(days=2) end = end or available_end self._log.info( @@ -628,7 +616,7 @@ async def _request_instruments( dataset: Dataset = self._loader.get_dataset_for_venue(venue) _, available_end = await self._get_dataset_range(dataset) - start = start or available_end - ONE_DAY * 2 + start = start or available_end - pd.Timedelta(days=2) end = end or available_end self._log.info( @@ -663,7 +651,7 @@ async def _request_quote_ticks( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) _, available_end = await self._get_dataset_range(dataset) - start = start or available_end - ONE_DAY + start = start or available_end - pd.Timedelta(days=1) end = end or available_end if limit > 0: @@ -703,7 +691,7 @@ async def _request_trade_ticks( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) _, available_end = await self._get_dataset_range(dataset) - start = start or available_end - ONE_DAY + start = start or available_end - pd.Timedelta(days=1) end = end or available_end if limit > 0: @@ -720,7 +708,7 @@ async def _request_trade_ticks( pyo3_trades = await self._http_client.get_range_trades( dataset=dataset, symbols=instrument_id.symbol.value, - start=(start or available_end - ONE_DAY).value, + start=(start or available_end - pd.Timedelta(days=1)).value, end=(end or available_end).value, ) @@ -743,7 +731,7 @@ async def _request_bars( dataset: Dataset = self._loader.get_dataset_for_venue(bar_type.instrument_id.venue) _, available_end = await self._get_dataset_range(dataset) - start = start or available_end - ONE_DAY + start = start or available_end - pd.Timedelta(days=1) end = end or available_end if limit > 0: @@ -763,7 +751,7 @@ async def _request_bars( aggregation=nautilus_pyo3.BarAggregation( bar_aggregation_to_str(bar_type.spec.aggregation), ), - start=(start or available_end - ONE_DAY).value, + start=(start or available_end - pd.Timedelta(days=1)).value, end=(end or available_end).value, ) diff --git a/nautilus_trader/adapters/databento/factories.py b/nautilus_trader/adapters/databento/factories.py index 5928e51dfc56..6238b3d97d3f 100644 --- a/nautilus_trader/adapters/databento/factories.py +++ b/nautilus_trader/adapters/databento/factories.py @@ -15,9 +15,9 @@ import asyncio from functools import lru_cache -from pathlib import Path from nautilus_trader.adapters.databento.config import DatabentoDataClientConfig +from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH from nautilus_trader.adapters.databento.data import DatabentoDataClient from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.adapters.databento.providers import DatabentoInstrumentProvider @@ -60,11 +60,12 @@ def get_cached_databento_http_client( key = key or get_env_key("DATABENTO_API_KEY") - publishers_path = str((Path(__file__).resolve().parent / "publishers.json").resolve()) - client_key: str = "|".join((key, gateway or "")) if client_key not in DATABENTO_HTTP_CLIENTS: - client = nautilus_pyo3.DatabentoHistoricalClient(key=key, publishers_path=publishers_path) + client = nautilus_pyo3.DatabentoHistoricalClient( + key=key, + publishers_path=str(PUBLISHERS_PATH), + ) DATABENTO_HTTP_CLIENTS[client_key] = client return DATABENTO_HTTP_CLIENTS[client_key] diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index 42df78dc2f6b..2141815fa406 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -20,6 +20,7 @@ import msgspec from nautilus_trader.adapters.databento.common import check_file_path +from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH from nautilus_trader.adapters.databento.types import DatabentoPublisher from nautilus_trader.core import nautilus_pyo3 from nautilus_trader.core.data import Data @@ -74,12 +75,10 @@ def __init__(self) -> None: self._publishers: dict[int, DatabentoPublisher] = {} self._instruments: dict[InstrumentId, Instrument] = {} - publishers_path = Path(__file__).resolve().parent / "publishers.json" - self._pyo3_loader: nautilus_pyo3.DatabentoDataLoader = nautilus_pyo3.DatabentoDataLoader( - str(publishers_path.resolve()), + str(PUBLISHERS_PATH), ) - self.load_publishers(path=publishers_path) + self.load_publishers(path=PUBLISHERS_PATH) @property def publishers(self) -> dict[int, DatabentoPublisher]: From 9462a840753c7ef97a4927da058df829caa3596d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 09:34:10 +1100 Subject: [PATCH 12/42] Add Databento asyncio.CancelledError handling --- nautilus_trader/adapters/databento/data.py | 284 ++++++++++++--------- 1 file changed, 162 insertions(+), 122 deletions(-) diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index d530fbd8ca67..383862e4dc32 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -276,14 +276,19 @@ def _send_all_instruments_to_data_engine(self) -> None: self._handle_data(instrument) async def _ensure_subscribed_for_instrument(self, instrument_id: InstrumentId) -> None: - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - subscribed_instruments = self._instrument_ids[dataset] + try: + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + subscribed_instruments = self._instrument_ids[dataset] - if instrument_id in subscribed_instruments: - return + if instrument_id in subscribed_instruments: + return - self._instrument_ids[dataset].add(instrument_id) - await self._subscribe_instrument(instrument_id) + self._instrument_ids[dataset].add(instrument_id) + await self._subscribe_instrument(instrument_id) + except asyncio.CancelledError: + self._log.warning( + "`_ensure_subscribed_for_instrument` was canceled while still pending.", + ) async def _get_dataset_range( self, @@ -314,6 +319,9 @@ async def _get_dataset_range( ) return available_start, available_end + except asyncio.CancelledError: + self._log.warning("`_get_dataset_range` was canceled while still pending.") + return (None, pd.Timestamp.utcnow()) except Exception as e: # More specific exception self._log.error(f"Error requesting dataset range: {e}") return (None, pd.Timestamp.utcnow()) @@ -354,38 +362,47 @@ async def _subscribe_instruments(self) -> None: raise NotImplementedError("Cannot subscribe to all instruments (not currently supported).") async def _subscribe_instrument(self, instrument_id: InstrumentId) -> None: - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="definition", - symbols=instrument_id.symbol.value, - ) - self._check_live_client_started(dataset, live_client) + try: + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema="definition", + symbols=instrument_id.symbol.value, + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_instrument` was canceled while still pending.") async def _subscribe_parent_symbols( self, dataset: Dataset, parent_symbols: set[str], ) -> None: - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="definition", - symbols=",".join(sorted(parent_symbols)), - stype_in="parent", - ) - self._check_live_client_started(dataset, live_client) + try: + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema="definition", + symbols=",".join(sorted(parent_symbols)), + stype_in="parent", + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_parent_symbols` was canceled while still pending.") async def _subscribe_instrument_ids( self, dataset: Dataset, instrument_ids: list[InstrumentId], ) -> None: - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="definition", - symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), - ) - self._check_live_client_started(dataset, live_client) + try: + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema="definition", + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_instrument_ids` was canceled while still pending.") async def _subscribe_order_book_deltas( self, @@ -394,64 +411,75 @@ async def _subscribe_order_book_deltas( depth: int | None = None, kwargs: dict | None = None, ) -> None: - if book_type != BookType.L3_MBO: - raise NotImplementedError + try: + if book_type != BookType.L3_MBO: + raise NotImplementedError - if depth: # Can be None or 0 (full depth) - self._log.error( - f"Cannot subscribe to order book deltas with specific depth of {depth} " - "(do not specify depth when subscribing, must be full depth).", - ) - return + if depth: # Can be None or 0 (full depth) + self._log.error( + f"Cannot subscribe to order book deltas with specific depth of {depth} " + "(do not specify depth when subscribing, must be full depth).", + ) + return - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - if self._is_buffering_mbo_subscriptions: - self._log.debug(f"Buffering MBO/L3 subscription for {instrument_id}.", LogColor.MAGENTA) - self._buffered_mbo_subscriptions[dataset].append(instrument_id) - return + if self._is_buffering_mbo_subscriptions: + self._log.debug( + f"Buffering MBO/L3 subscription for {instrument_id}.", + LogColor.MAGENTA, + ) + self._buffered_mbo_subscriptions[dataset].append(instrument_id) + return - if self._get_live_client_mbo(dataset) is not None: - self._log.error( - f"Cannot subscribe to order book deltas for {instrument_id}, " - "MBO/L3 feed already started.", - ) - return + if self._get_live_client_mbo(dataset) is not None: + self._log.error( + f"Cannot subscribe to order book deltas for {instrument_id}, " + "MBO/L3 feed already started.", + ) + return - await self._subscribe_order_book_deltas_batch([instrument_id]) + await self._subscribe_order_book_deltas_batch([instrument_id]) + except asyncio.CancelledError: + self._log.warning("`_subscribe_order_book_deltas` was canceled while still pending.") async def _subscribe_order_book_deltas_batch( self, instrument_ids: list[InstrumentId], ) -> None: - if not instrument_ids: + try: + if not instrument_ids: + self._log.warning( + "No subscriptions for order book deltas (`instrument_ids` was empty).", + ) + return + + for instrument_id in instrument_ids: + if not self._cache.instrument(instrument_id): + self._log.error( + f"Cannot subscribe to order book deltas for {instrument_id}, " + "instrument must be pre-loaded via the `DatabentoDataClientConfig` " + "or a specific subscription on start.", + ) + instrument_ids.remove(instrument_id) + continue + + if not instrument_ids: + return # No subscribing instrument IDs were loaded in the cache + + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_ids[0].venue) + live_client = self._get_live_client_mbo(dataset) + await live_client.subscribe( + schema="mbo", + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), + start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now + ) + future = asyncio.ensure_future(live_client.start(self._handle_record)) + self._live_client_futures.add(future) + except asyncio.CancelledError: self._log.warning( - "No subscriptions for order book deltas (`instrument_ids` was empty).", + "`_subscribe_order_book_deltas_batch` was canceled while still pending.", ) - return - - for instrument_id in instrument_ids: - if not self._cache.instrument(instrument_id): - self._log.error( - f"Cannot subscribe to order book deltas for {instrument_id}, " - "instrument must be pre-loaded via the `DatabentoDataClientConfig` " - "or a specific subscription on start.", - ) - instrument_ids.remove(instrument_id) - continue - - if not instrument_ids: - return # No subscribing instrument IDs were loaded in the cache - - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_ids[0].venue) - live_client = self._get_live_client_mbo(dataset) - await live_client.subscribe( - schema="mbo", - symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), - start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now - ) - future = asyncio.ensure_future(live_client.start(self._handle_record)) - self._live_client_futures.add(future) async def _subscribe_order_book_snapshots( self, @@ -460,67 +488,79 @@ async def _subscribe_order_book_snapshots( depth: int | None = None, kwargs: dict | None = None, ) -> None: - await self._ensure_subscribed_for_instrument(instrument_id) - - match depth: - case 1: - schema = "mbp-1" - case 10: - schema = "mbp-10" - case _: - self._log.error( - f"Cannot subscribe for order book snapshots of depth {depth}, use either 1 or 10.", - ) - return - - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema=schema, - symbols=",".join(sorted([instrument_id.symbol.value])), - ) - self._check_live_client_started(dataset, live_client) + try: + await self._ensure_subscribed_for_instrument(instrument_id) + + match depth: + case 1: + schema = "mbp-1" + case 10: + schema = "mbp-10" + case _: + self._log.error( + f"Cannot subscribe for order book snapshots of depth {depth}, use either 1 or 10.", + ) + return + + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema=schema, + symbols=",".join(sorted([instrument_id.symbol.value])), + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_order_book_snapshots` was canceled while still pending.") async def _subscribe_quote_ticks(self, instrument_id: InstrumentId) -> None: - await self._ensure_subscribed_for_instrument(instrument_id) + try: + await self._ensure_subscribed_for_instrument(instrument_id) - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="mbp-1", - symbols=",".join(sorted([instrument_id.symbol.value])), - ) - self._check_live_client_started(dataset, live_client) + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema="mbp-1", + symbols=",".join(sorted([instrument_id.symbol.value])), + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_quote_ticks` was canceled while still pending.") async def _subscribe_trade_ticks(self, instrument_id: InstrumentId) -> None: - if instrument_id in self.subscribed_quote_ticks(): - return # Already subscribed for trades + try: + if instrument_id in self.subscribed_quote_ticks(): + return # Already subscribed for trades - await self._ensure_subscribed_for_instrument(instrument_id) + await self._ensure_subscribed_for_instrument(instrument_id) - dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="trades", - symbols=instrument_id.symbol.value, - ) - self._check_live_client_started(dataset, live_client) + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema="trades", + symbols=instrument_id.symbol.value, + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_trade_ticks` was canceled while still pending.") async def _subscribe_bars(self, bar_type: BarType) -> None: - dataset: Dataset = self._loader.get_dataset_for_venue(bar_type.instrument_id.venue) - try: - schema = databento_schema_from_nautilus_bar_type(bar_type) - except ValueError as e: - self._log.error(f"Cannot subscribe: {e}") - return + dataset: Dataset = self._loader.get_dataset_for_venue(bar_type.instrument_id.venue) - live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema=schema, - symbols=bar_type.instrument_id.symbol.value, - ) - self._check_live_client_started(dataset, live_client) + try: + schema = databento_schema_from_nautilus_bar_type(bar_type) + except ValueError as e: + self._log.error(f"Cannot subscribe: {e}") + return + + live_client = self._get_live_client(dataset) + await live_client.subscribe( + schema=schema, + symbols=bar_type.instrument_id.symbol.value, + ) + self._check_live_client_started(dataset, live_client) + except asyncio.CancelledError: + self._log.warning("`_subscribe_bars` was canceled while still pending.") async def _unsubscribe(self, data_type: DataType) -> None: raise NotImplementedError( From 5bf5a380c4ee848dffe9cbd9dea6fd7374ace201 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 11:20:12 +1100 Subject: [PATCH 13/42] Refine Databento adapter --- docs/getting_started/installation.md | 3 +- .../adapters/src/databento/loader.rs | 4 +- .../adapters/src/databento/python/live.rs | 33 +- .../adapters/src/databento/python/loader.rs | 2 +- nautilus_core/pyo3/src/lib.rs | 9 +- nautilus_trader/adapters/databento/data.py | 28 +- nautilus_trader/adapters/databento/enums.py | 57 ++ nautilus_trader/adapters/databento/loaders.py | 24 +- nautilus_trader/adapters/databento/parsing.py | 553 ------------------ .../adapters/databento/providers.py | 64 +- nautilus_trader/core/nautilus_pyo3.pyi | 5 +- poetry.lock | 209 +------ pyproject.toml | 2 - 13 files changed, 147 insertions(+), 846 deletions(-) delete mode 100644 nautilus_trader/adapters/databento/parsing.py diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 525976acc209..58b4015b7f2b 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -22,8 +22,7 @@ To install the latest binary wheel (or sdist package) from PyPI using Pythons _p Install optional dependencies as 'extras' for specific integrations: - `betfair`: Betfair adapter (integration) -- `databento`: Databento adapter (integration) -- `docker`: Needed for Docker when using the IB gateway +- `docker`: Needed for Docker when using the IB gateway (with the Interactive Brokers adapter) - `ib`: Interactive Brokers adapter To install with specific extras using _pip_: diff --git a/nautilus_core/adapters/src/databento/loader.rs b/nautilus_core/adapters/src/databento/loader.rs index 56cebff5a15c..2b33f68d1043 100644 --- a/nautilus_core/adapters/src/databento/loader.rs +++ b/nautilus_core/adapters/src/databento/loader.rs @@ -193,10 +193,10 @@ impl DatabentoDataLoader { Ok(InstrumentId::new(symbol, venue)) } - pub fn schema_from_file(&self, path: PathBuf) -> Result> { + pub fn schema_from_file(&self, path: PathBuf) -> Result> { let decoder = Decoder::from_zstd_file(path)?; let metadata = decoder.metadata(); - Ok(metadata.schema) + Ok(metadata.schema.map(|schema| schema.to_string())) } pub fn read_records( diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 8b02401d8ddb..1c50f0169dff 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -35,9 +35,11 @@ use pyo3::prelude::*; use time::OffsetDateTime; use tokio::sync::Mutex; -use crate::databento::parsing::parse_record; +use crate::databento::parsing::{parse_instrument_def_msg, parse_record}; use crate::databento::types::{DatabentoPublisher, PublisherId}; +use super::loader::convert_instrument_to_pyobject; + #[cfg_attr( feature = "python", pyclass(module = "nautilus_trader.core.nautilus_pyo3.databento") @@ -166,6 +168,31 @@ impl DatabentoLiveClient { error!("{:?}", record); continue; } + RType::InstrumentDef => { + let msg = record + .get::() + .expect("Error converting record to `InstrumentDefMsg`"); + let publisher_id = record.publisher().unwrap() as PublisherId; + let publisher = publishers.get(&publisher_id).unwrap(); + let ts_init = clock.get_time_ns(); + let result = parse_instrument_def_msg(msg, publisher, ts_init); + + match result { + Ok(instrument) => { + // TODO: Improve the efficiency of this constant GIL aquisition + Python::with_gil(|py| { + let py_obj = + convert_instrument_to_pyobject(py, instrument).unwrap(); + match callback.call1(py, (py_obj,)) { + Ok(_) => {} + Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now + }; + }); + } + Err(e) => eprintln!("{:?}", e), + } + continue; + } _ => {} // Fall through } @@ -186,7 +213,7 @@ impl DatabentoLiveClient { // TODO: Improve the efficiency of this constant GIL aquisition Python::with_gil(|py| { - let data = match data { + let py_obj = match data { Data::Delta(delta) => delta.into_py(py), Data::Depth10(depth) => depth.into_py(py), Data::Quote(quote) => quote.into_py(py), @@ -194,7 +221,7 @@ impl DatabentoLiveClient { _ => panic!("Invalid data element, was {:?}", data), }; - match callback.call1(py, (data,)) { + match callback.call1(py, (py_obj,)) { Ok(_) => {} Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now }; diff --git a/nautilus_core/adapters/src/databento/python/loader.rs b/nautilus_core/adapters/src/databento/python/loader.rs index f273101017fb..aac731c6454e 100644 --- a/nautilus_core/adapters/src/databento/python/loader.rs +++ b/nautilus_core/adapters/src/databento/python/loader.rs @@ -39,7 +39,7 @@ impl DatabentoDataLoader { } #[pyo3(name = "schema_for_file")] - pub fn py_schema_for_file(&self, path: String) -> PyResult> { + pub fn py_schema_for_file(&self, path: String) -> PyResult> { self.schema_from_file(PathBuf::from(path)) .map_err(to_pyvalue_err) } diff --git a/nautilus_core/pyo3/src/lib.rs b/nautilus_core/pyo3/src/lib.rs index 44fcafc5de7e..ea28254a9125 100644 --- a/nautilus_core/pyo3/src/lib.rs +++ b/nautilus_core/pyo3/src/lib.rs @@ -14,12 +14,7 @@ // ------------------------------------------------------------------------------------------------- use nautilus_adapters::databento::{ - common::{ALL_SYMBOLS, DATABENTO}, - loader, - python::historical, - python::live, - python::parsing, - types, + loader, python::historical, python::live, python::parsing, types, }; use pyo3::{ prelude::*, @@ -32,8 +27,6 @@ use pyo3::{ /// Loaded as nautilus_pyo3.databento #[pymodule] pub fn databento(_: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add(stringify!(DATABENTO), DATABENTO)?; - m.add(stringify!(ALL_SYMBOLS), ALL_SYMBOLS)?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 383862e4dc32..c3695b2de923 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -18,7 +18,6 @@ from collections.abc import Coroutine from typing import Any -import databento_dbn import pandas as pd import pytz @@ -27,6 +26,7 @@ from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH +from nautilus_trader.adapters.databento.enums import DatabentoRecordFlags from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.adapters.databento.providers import DatabentoInstrumentProvider from nautilus_trader.adapters.databento.types import Dataset @@ -810,23 +810,11 @@ def _handle_record( ) -> None: # self._log.debug(f"Received {record}", LogColor.MAGENTA) - # TODO: Handle inside client - - # if isinstance(record, databento.ErrorMsg): - # self._log.error(f"ErrorMsg: {record.err}") - # return - # elif isinstance(record, databento.SystemMsg): - # self._log.info(f"SystemMsg: {record.msg}") - # return - # elif isinstance(record, databento.SymbolMappingMsg): - # self._log.debug(f"SymbolMappingMsg: {record}") - # return - - instrument_id = InstrumentId.from_str(pyo3_data.instrument_id.value) - if isinstance(pyo3_data, nautilus_pyo3.OrderBookDelta): + instrument_id = InstrumentId.from_str(pyo3_data.instrument_id.value) + data = OrderBookDelta.from_pyo3_list([pyo3_data]) # TODO: Implement single function - if databento_dbn.RecordFlags.F_LAST not in databento_dbn.RecordFlags(data.flags): + if DatabentoRecordFlags.F_LAST not in DatabentoRecordFlags(data.flags): buffer = self._buffered_deltas[instrument_id] buffer.append(data) return # We can rely on the F_LAST flag for an MBO feed @@ -836,13 +824,13 @@ def _handle_record( data = OrderBookDeltas(instrument_id, deltas=buffer.copy()) buffer.clear() elif isinstance(pyo3_data, nautilus_pyo3.OrderBookDepth10): - data = OrderBookDepth10.from_pyo3_list([pyo3_data]) + data = OrderBookDepth10.from_pyo3_list([pyo3_data]) # TODO: Implement single function elif isinstance(pyo3_data, nautilus_pyo3.QuoteTick): - data = QuoteTick.from_pyo3_list([pyo3_data]) + data = QuoteTick.from_pyo3_list([pyo3_data]) # TODO: Implement single function elif isinstance(pyo3_data, nautilus_pyo3.TradeTick): - data = TradeTick.from_pyo3_list([pyo3_data]) + data = TradeTick.from_pyo3_list([pyo3_data]) # TODO: Implement single function elif isinstance(pyo3_data, nautilus_pyo3.Bar): - data = Bar.from_pyo3_list([pyo3_data]) + data = Bar.from_pyo3_list([pyo3_data]) # TODO: Implement single function else: self._log.error(f"Unimplemented data type in handler: {pyo3_data}.") return diff --git a/nautilus_trader/adapters/databento/enums.py b/nautilus_trader/adapters/databento/enums.py index 0e7d1a2c9cca..d28f2ce9a817 100644 --- a/nautilus_trader/adapters/databento/enums.py +++ b/nautilus_trader/adapters/databento/enums.py @@ -14,11 +14,60 @@ # ------------------------------------------------------------------------------------------------- from enum import Enum +from enum import IntFlag from enum import unique +class DatabentoSchema(Enum): + """ + Represents a Databento schema. + """ + + MBO = "mbo" + MBP_1 = "mbp-1" + MBP_10 = "mbp-10" + TBBO = "tbbo" + TRADES = "trades" + OHLCV_1S = "ohlcv-1s" + OHLCV_1M = "ohlcv-1m" + OHLCV_1H = "ohlcv-1h" + OHLCV_1D = "ohlcv-1d" + OHLCV_EOD = "ohlcv-eod" + DEFINITION = "definition" + STATISTICS = "statistics" + STATUS = "status" + IMBALANCE = "imbalance" + + +class DatabentoRecordFlags(IntFlag): + """ + Represents Databento record flags. + + F_LAST + Last message in the packet from the venue for a given Databento `instrument_id`. + F_SNAPSHOT + Message sourced from a replay, such as a snapshot server. + F_MBP + Aggregated price level message, not an individual order. + F_BAD_TS_RECV + The `ts_recv` value is inaccurate (clock issues or reordering). + + Other bits are reserved and have no current meaning. + + """ + + F_LAST = 128 + F_SNAPSHOT = 32 + F_MBP = 16 + F_BAD_TS_RECV = 8 + + @unique class DatabentoInstrumentClass(Enum): + """ + Represents a Databento instrument class. + """ + BOND = "B" CALL = "C" FUTURE = "F" @@ -32,6 +81,10 @@ class DatabentoInstrumentClass(Enum): @unique class DatabentoStatisticType(Enum): + """ + Represents a Databento statistic type. + """ + OPENING_PRICE = 1 INDICATIVE_OPENING_PRICE = 2 SETTLEMENT_PRICE = 3 @@ -48,5 +101,9 @@ class DatabentoStatisticType(Enum): @unique class DatabentoStatisticUpdateAction(Enum): + """ + Represents a Databento statistic update action. + """ + ADDED = 1 DELETED = 2 diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index 2141815fa406..3a115b26d4ba 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -16,11 +16,11 @@ from os import PathLike from pathlib import Path -import databento_dbn import msgspec from nautilus_trader.adapters.databento.common import check_file_path from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH +from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.adapters.databento.types import DatabentoPublisher from nautilus_trader.core import nautilus_pyo3 from nautilus_trader.core.data import Data @@ -183,42 +183,42 @@ def load_from_file_pyo3( else None ) - schema = self._pyo3_loader.schema_for_file(path) # type: ignore + schema = self._pyo3_loader.schema_for_file(str(path)) if schema is None: raise RuntimeError("Loading files with mixed schemas not currently supported") match schema: - case databento_dbn.Schema.DEFINITION: + case DatabentoSchema.DEFINITION.value: data = self._pyo3_loader.load_instruments(path) # type: ignore if as_legacy_cython: data = instruments_from_pyo3(data) return data - case databento_dbn.Schema.MBO: + case DatabentoSchema.MBO.value: data = self._pyo3_loader.load_order_book_deltas(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: data = OrderBookDelta.from_pyo3_list(data) return data - case databento_dbn.Schema.MBP_1 | databento_dbn.Schema.TBBO: + case DatabentoSchema.MBP_1.value | DatabentoSchema.TBBO.value: data = self._pyo3_loader.load_quote_ticks(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: data = QuoteTick.from_pyo3_list(data) return data - case databento_dbn.Schema.MBP_10: + case DatabentoSchema.MBP_10.value: data = self._pyo3_loader.load_order_book_depth10(path) # type: ignore if as_legacy_cython: data = OrderBookDepth10.from_pyo3_list(data) return data - case databento_dbn.Schema.TRADES: + case DatabentoSchema.TRADES.value: data = self._pyo3_loader.load_trade_ticks(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: data = TradeTick.from_pyo3_list(data) return data case ( - databento_dbn.Schema.OHLCV_1S - | databento_dbn.Schema.OHLCV_1M - | databento_dbn.Schema.OHLCV_1H - | databento_dbn.Schema.OHLCV_1D - | databento_dbn.Schema.OHLCV_EOD + DatabentoSchema.OHLCV_1S.value + | DatabentoSchema.OHLCV_1M.value + | DatabentoSchema.OHLCV_1H.value + | DatabentoSchema.OHLCV_1D.value + | DatabentoSchema.OHLCV_EOD ): data = self._pyo3_loader.load_bars(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: diff --git a/nautilus_trader/adapters/databento/parsing.py b/nautilus_trader/adapters/databento/parsing.py deleted file mode 100644 index 6e83aafdd6bc..000000000000 --- a/nautilus_trader/adapters/databento/parsing.py +++ /dev/null @@ -1,553 +0,0 @@ -# ------------------------------------------------------------------------------------------------- -# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. -# https://nautechsystems.io -# -# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ------------------------------------------------------------------------------------------------- - -import databento -import pandas as pd -import pytz - -from nautilus_trader.adapters.databento.common import nautilus_instrument_id_from_databento -from nautilus_trader.adapters.databento.enums import DatabentoInstrumentClass -from nautilus_trader.adapters.databento.enums import DatabentoStatisticType -from nautilus_trader.adapters.databento.enums import DatabentoStatisticUpdateAction -from nautilus_trader.adapters.databento.types import DatabentoImbalance -from nautilus_trader.adapters.databento.types import DatabentoPublisher -from nautilus_trader.adapters.databento.types import DatabentoStatistics -from nautilus_trader.core.data import Data -from nautilus_trader.core.datetime import secs_to_nanos -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.data import Bar -from nautilus_trader.model.data import BarSpecification -from nautilus_trader.model.data import BarType -from nautilus_trader.model.data import BookOrder -from nautilus_trader.model.data import OrderBookDelta -from nautilus_trader.model.data import OrderBookDepth10 -from nautilus_trader.model.data import QuoteTick -from nautilus_trader.model.data import TradeTick -from nautilus_trader.model.enums import AggregationSource -from nautilus_trader.model.enums import AggressorSide -from nautilus_trader.model.enums import AssetClass -from nautilus_trader.model.enums import BarAggregation -from nautilus_trader.model.enums import BookAction -from nautilus_trader.model.enums import InstrumentClass -from nautilus_trader.model.enums import OptionKind -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import PriceType -from nautilus_trader.model.identifiers import InstrumentId -from nautilus_trader.model.identifiers import Symbol -from nautilus_trader.model.identifiers import TradeId -from nautilus_trader.model.instruments import Equity -from nautilus_trader.model.instruments import FuturesContract -from nautilus_trader.model.instruments import Instrument -from nautilus_trader.model.instruments import OptionsContract -from nautilus_trader.model.objects import FIXED_SCALAR -from nautilus_trader.model.objects import Currency -from nautilus_trader.model.objects import Price -from nautilus_trader.model.objects import Quantity - - -def parse_order_side(value: str) -> OrderSide: - match value: - case "A": - return OrderSide.SELL - case "B": - return OrderSide.BUY - case _: - return OrderSide.NO_ORDER_SIDE - - -def parse_aggressor_side(value: str) -> AggressorSide: - match value: - case "A": - return AggressorSide.SELLER - case "B": - return AggressorSide.BUYER - case _: - return AggressorSide.NO_AGGRESSOR - - -def parse_book_action(value: str) -> BookAction: - match value: - case "A": - return BookAction.ADD - case "C": - return BookAction.DELETE - case "F": - return BookAction.UPDATE - case "M": - return BookAction.UPDATE - case "R": - return BookAction.CLEAR - case _: - raise ValueError(f"Invalid `BookAction`, was '{value}'") - - -def parse_option_kind(value: str) -> OptionKind: - match value: - case "C": - return OptionKind.CALL - case "P": - return OptionKind.PUT - case _: - raise ValueError(f"Invalid `OptionKind`, was '{value}'") - - -def parse_cfi_iso10926(value: str) -> tuple[AssetClass | None, InstrumentClass | None]: - # This is a work in progress and will likely result in a shuffling of - # the `AssetClass` and `InstrumentClass` enums - - cfi_category = value[0] - cfi_group = value[1] - cfi_attribute1 = value[2] - # cfi_attribute2 = value[3] - # cfi_attribute3 = value[4] - # cfi_attribute4 = value[5] - - match cfi_category: - case "D": - asset_class = AssetClass.DEBT - case "E": - asset_class = AssetClass.EQUITY - case "S": - asset_class = None - instrument_class = InstrumentClass.SWAP - case "J": - asset_class = None - instrument_class = InstrumentClass.FORWARD - case _: - asset_class = None - - match cfi_group: - case "I": - instrument_class = InstrumentClass.FUTURE - case _: - instrument_class = None - - match cfi_attribute1: - case "I": - asset_class = AssetClass.INDEX - - return (asset_class, instrument_class) - - -def parse_min_price_increment(value: int, currency: Currency) -> Price: - match value: - case 0 | 9223372036854775807: # 2**63-1 (TODO: Make limit constants) - return Price(10 ** (-currency.precision), currency.precision) - case _: - return Price.from_raw(value, currency.precision) - - -def parse_equity( - record: databento.InstrumentDefMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> Equity: - # Use USD for all US equities venues for now - currency = USD - - return Equity( - instrument_id=instrument_id, - raw_symbol=Symbol(record.raw_symbol), - currency=currency, - price_precision=currency.precision, - price_increment=parse_min_price_increment(record.min_price_increment, currency), - lot_size=Quantity.from_int(record.min_lot_size_round_lot), - isin=None, # TODO - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_futures_contract( - record: databento.InstrumentDefMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> FuturesContract: - currency = Currency.from_str(record.currency) - asset_class, _ = parse_cfi_iso10926(record.cfi) - - return FuturesContract( - instrument_id=instrument_id, - raw_symbol=Symbol(record.raw_symbol), - asset_class=asset_class or AssetClass.COMMODITY, # WIP - currency=currency, - price_precision=currency.precision, - price_increment=parse_min_price_increment(record.min_price_increment, currency), - multiplier=Quantity(1, precision=0), - lot_size=Quantity(record.min_lot_size_round_lot or 1, precision=0), - underlying=record.asset, - activation_ns=record.activation, - expiration_ns=record.expiration, - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_options_contract( - record: databento.InstrumentDefMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> OptionsContract: - currency = Currency.from_str(record.currency) - - if instrument_id.venue.value == "OPRA": - lot_size = Quantity(1, precision=0) - asset_class = AssetClass.EQUITY - else: - lot_size = Quantity(record.min_lot_size_round_lot or 1, precision=0) - asset_class, _ = parse_cfi_iso10926(record.cfi) - asset_class = asset_class or AssetClass.EQUITY # WIP - - return OptionsContract( - instrument_id=instrument_id, - raw_symbol=Symbol(record.raw_symbol), - asset_class=asset_class, - currency=currency, - price_precision=currency.precision, - price_increment=parse_min_price_increment(record.min_price_increment, currency), - multiplier=Quantity(1, precision=0), - lot_size=lot_size, - underlying=record.underlying, - kind=parse_option_kind(record.instrument_class), - activation_ns=record.activation, - expiration_ns=record.expiration, - strike_price=Price.from_raw(record.strike_price, currency.precision), - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_mbo_msg( - record: databento.MBOMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> OrderBookDelta | TradeTick: - side: OrderSide = parse_order_side(record.side) - if side == OrderSide.NO_ORDER_SIDE or record.action == "T": - return TradeTick.from_raw( - instrument_id=instrument_id, - price_raw=record.price, - price_prec=USD.precision, # TODO(per instrument precision) - size_raw=int(record.size * FIXED_SCALAR), # No fractional sizes - size_prec=0, # No fractional units - aggressor_side=parse_aggressor_side(record.side), - trade_id=TradeId(str(record.sequence)), - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - return OrderBookDelta.from_raw( - instrument_id=instrument_id, - action=parse_book_action(record.action), - side=side, - price_raw=record.price, - price_prec=USD.precision, # TODO(per instrument precision) - size_raw=int(record.size * FIXED_SCALAR), # No fractional sizes - size_prec=0, # No fractional units - order_id=record.order_id, - flags=record.flags, - sequence=record.sequence, - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_mbp1_msg( - record: databento.MBP1Msg, - instrument_id: InstrumentId, - ts_init: int, -) -> QuoteTick | tuple[QuoteTick | TradeTick]: - top_level = record.levels[0] - quote = QuoteTick.from_raw( - instrument_id=instrument_id, - bid_price_raw=top_level.bid_px, - bid_price_prec=USD.precision, # TODO(per instrument precision) - ask_price_raw=top_level.ask_px, - ask_price_prec=USD.precision, # TODO(per instrument precision) - bid_size_raw=int(top_level.bid_sz * FIXED_SCALAR), # No fractional sizes - bid_size_prec=0, # No fractional units - ask_size_raw=int(top_level.ask_sz * FIXED_SCALAR), # No fractional sizes - ask_size_prec=0, # No fractional units - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - match record.action: - case "T": - trade = TradeTick.from_raw( - instrument_id=instrument_id, - price_raw=record.price, - price_prec=USD.precision, # TODO(per instrument precision) - size_raw=int(record.size * FIXED_SCALAR), # No fractional sizes - size_prec=0, # No fractional units - aggressor_side=parse_aggressor_side(record.side), - trade_id=TradeId(str(record.sequence)), - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - return quote, trade - case _: - return quote - - -def parse_mbp10_msg( - record: databento.MBP10Msg, - instrument_id: InstrumentId, - ts_init: int, -) -> OrderBookDepth10: - bids: list[BookOrder] = [] - asks: list[BookOrder] = [] - bid_counts: list[int] = [] - ask_counts: list[int] = [] - - for level in record.levels: - bid = BookOrder.from_raw( - side=OrderSide.BUY, - price_raw=level.bid_px, - price_prec=USD.precision, - size_raw=int(level.bid_sz * FIXED_SCALAR), - size_prec=0, # No fractional units - order_id=0, # No order ID for MBP level - ) - bids.append(bid) - bid_counts.append(level.bid_ct) - - ask = BookOrder.from_raw( - side=OrderSide.SELL, - price_raw=level.ask_px, - price_prec=USD.precision, - size_raw=int(level.ask_sz * FIXED_SCALAR), - size_prec=0, # No fractional units - order_id=0, # No order ID for MBP level - ) - asks.append(ask) - ask_counts.append(level.ask_ct) # Currently a typo in type stub # type: ignore - - return OrderBookDepth10( - instrument_id=instrument_id, - bids=bids, - asks=asks, - bid_counts=bid_counts, - ask_counts=ask_counts, - flags=record.flags, - sequence=record.sequence, - ts_event=record.ts_recv, - ts_init=ts_init, - ) - - -def parse_trade_msg( - record: databento.TradeMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> TradeTick: - return TradeTick.from_raw( - instrument_id=instrument_id, - price_raw=record.price, - price_prec=USD.precision, # TODO(per instrument precision) - size_raw=int(record.size * FIXED_SCALAR), - size_prec=0, # No fractional units - aggressor_side=parse_aggressor_side(record.side), - trade_id=TradeId(str(record.sequence)), - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_ohlcv_msg( - record: databento.OHLCVMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> Bar: - match record.rtype: - case 32: # ohlcv-1s - bar_spec = BarSpecification(1, BarAggregation.SECOND, PriceType.LAST) - bar_type = BarType(instrument_id, bar_spec, AggregationSource.EXTERNAL) - ts_event_adjustment = secs_to_nanos(1) - case 33: # ohlcv-1m - bar_spec = BarSpecification(1, BarAggregation.MINUTE, PriceType.LAST) - bar_type = BarType(instrument_id, bar_spec, AggregationSource.EXTERNAL) - ts_event_adjustment = secs_to_nanos(60) - case 34: # ohlcv-1h - bar_spec = BarSpecification(1, BarAggregation.HOUR, PriceType.LAST) - bar_type = BarType(instrument_id, bar_spec, AggregationSource.EXTERNAL) - ts_event_adjustment = secs_to_nanos(60 * 60) - case 35: # ohlcv-1d - bar_spec = BarSpecification(1, BarAggregation.DAY, PriceType.LAST) - bar_type = BarType(instrument_id, bar_spec, AggregationSource.EXTERNAL) - ts_event_adjustment = secs_to_nanos(60 * 60 * 24) - case _: - raise ValueError("`rtype` is not a supported bar aggregation") - - # Adjust `ts_event` from open to close of bar - ts_event = record.ts_event + ts_event_adjustment - ts_init = max(ts_init, ts_event) - - return Bar( - bar_type=bar_type, - open=Price.from_raw(record.open / 100, 2), # TODO(adjust for display factor) - high=Price.from_raw(record.high / 100, 2), # TODO(adjust for display factor) - low=Price.from_raw(record.low / 100, 2), # TODO(adjust for display factor) - close=Price.from_raw(record.close / 100, 2), # TODO(adjust for display factor) - volume=Quantity.from_raw(record.volume, 0), # TODO(adjust for display factor) - ts_event=ts_event, - ts_init=ts_init, - ) - - -def parse_imbalance_msg( - record: databento.ImbalanceMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> TradeTick: - return DatabentoImbalance( - instrument_id=instrument_id, - ref_price=Price.from_raw(record.ref_price, USD.precision), - cont_book_clr_price=Price.from_raw(record.cont_book_clr_price, USD.precision), - auct_interest_clr_price=Price.from_raw(record.auct_interest_clr_price, USD.precision), - paired_qty=Quantity.from_int(record.paired_qty), # Always ints for now - total_imbalance_qty=Quantity.from_int(record.total_imbalance_qty), # Always ints for now - side=parse_order_side(record.side), - significant_imbalance=record.significant_imbalance, - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_statistics_msg( - record: databento.StatMsg, - instrument_id: InstrumentId, - ts_init: int, -) -> TradeTick: - return DatabentoStatistics( - instrument_id=instrument_id, - stat_type=DatabentoStatisticType(record.stat_type), - update_action=DatabentoStatisticUpdateAction(record.update_action), - price=Price.from_raw(record.price, USD.precision) - if record.price is not (2**63 - 1) # TODO: Define a constant for this - else None, - quantity=Quantity.from_raw(record.quantity, USD.precision) - if record.quantity is not (2**31 - 1) # TODO: Define a constant for this - else None, - channel_id=record.channel_id, - stat_flags=record.stat_flags, - sequence=record.sequence, - ts_ref=record.ts_ref, - ts_in_delta=record.ts_in_delta, - ts_event=record.ts_recv, # More accurate and reliable timestamp - ts_init=ts_init, - ) - - -def parse_instrument_id( - record: databento.DBNRecord, - publishers: dict[int, DatabentoPublisher], - instrument_map: databento.InstrumentMap, -) -> InstrumentId: - record_date = pd.Timestamp(record.ts_event, tz=pytz.utc).date() - raw_symbol = instrument_map.resolve(record.instrument_id, date=record_date) - if raw_symbol is None: - raise ValueError( - f"Cannot resolve instrument_id {record.instrument_id} on {record_date}", - ) - - publisher: DatabentoPublisher = publishers[record.publisher_id] - return nautilus_instrument_id_from_databento( - raw_symbol=raw_symbol, - publisher=publisher, - ) - - -def parse_record_with_metadata( - record: databento.DBNRecord, - publishers: dict[int, DatabentoPublisher], - ts_init: int, - instrument_map: databento.InstrumentMap | None = None, -) -> Data: - if isinstance(record, databento.InstrumentDefMsg): - return parse_instrument_def(record, publishers, ts_init) - - if instrument_map is None: - raise ValueError("`instrument_map` was `None` when a value was expected") - - record_date = pd.Timestamp(record.ts_event, tz=pytz.utc).date() - raw_symbol = instrument_map.resolve(record.instrument_id, date=record_date) - if raw_symbol is None: - raise ValueError( - f"Cannot resolve instrument_id {record.instrument_id} on {record_date}", - ) - - instrument_id: InstrumentId = parse_instrument_id(record, publishers, instrument_map) - - return parse_record( - record=record, - instrument_id=instrument_id, - ts_init=ts_init, - ) - - -def parse_record( - record: databento.DBNRecord, - instrument_id: InstrumentId, - ts_init: int, -) -> Data: - if isinstance(record, databento.MBOMsg): - return parse_mbo_msg(record, instrument_id, ts_init) - elif isinstance(record, databento.MBP1Msg): # Also TBBO - return parse_mbp1_msg(record, instrument_id, ts_init) - elif isinstance(record, databento.MBP10Msg): - return parse_mbp10_msg(record, instrument_id, ts_init) - elif isinstance(record, databento.TradeMsg): - return parse_trade_msg(record, instrument_id, ts_init) - elif isinstance(record, databento.OHLCVMsg): - return parse_ohlcv_msg(record, instrument_id, ts_init) - elif isinstance(record, databento.ImbalanceMsg): - return parse_imbalance_msg(record, instrument_id, ts_init) - elif isinstance(record, databento.StatMsg): - return parse_statistics_msg(record, instrument_id, ts_init) - else: - raise ValueError( - f"Schema {type(record).__name__} is currently unsupported by NautilusTrader", - ) - - -def parse_instrument_def( - record: databento.InstrumentDefMsg, - publishers: dict[int, DatabentoPublisher], - ts_init: int, -) -> Instrument: - publisher: DatabentoPublisher = publishers[record.publisher_id] - instrument_id: InstrumentId = nautilus_instrument_id_from_databento( - raw_symbol=record.raw_symbol, - publisher=publisher, - ) - - match record.instrument_class: - case DatabentoInstrumentClass.STOCK.value: - return parse_equity(record, instrument_id, ts_init) - case DatabentoInstrumentClass.FUTURE.value | DatabentoInstrumentClass.FUTURE_SPREAD.value: - return parse_futures_contract(record, instrument_id, ts_init) - case DatabentoInstrumentClass.CALL.value | DatabentoInstrumentClass.PUT.value: - return parse_options_contract(record, instrument_id, ts_init) - case DatabentoInstrumentClass.FUTURE_SPREAD.value: - raise ValueError("`instrument_class` FUTURE_SPREAD not currently supported") - case DatabentoInstrumentClass.OPTION_SPREAD.value: - raise ValueError("`instrument_class` OPTION_SPREAD not currently supported") - case DatabentoInstrumentClass.MIXED_SPREAD.value: - raise ValueError("`instrument_class` MIXED_SPREAD not currently supported") - case DatabentoInstrumentClass.FX_SPOT.value: - raise ValueError("`instrument_class` FX_SPOT not currently supported") - case _: - raise ValueError(f"Invalid `instrument_class`, was {record.instrument_class}") diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index 40eff48036f6..84c051df276e 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -13,15 +13,15 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- +import asyncio import datetime as dt -import databento import pandas as pd import pytz from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS +from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader -from nautilus_trader.adapters.databento.parsing import parse_record_with_metadata from nautilus_trader.common.component import LiveClock from nautilus_trader.common.providers import InstrumentProvider from nautilus_trader.config import InstrumentProviderConfig @@ -71,7 +71,6 @@ def __init__( self._live_gateway = live_gateway self._http_client = http_client - self._live_clients: dict[str, databento.Live] = {} self._loader = loader or DatabentoDataLoader() async def load_all_async(self, filters: dict | None = None) -> None: @@ -113,38 +112,37 @@ async def load_ids_async( """ PyCondition.not_empty(instrument_ids, "instrument_ids") - instrument_ids_to_decode = set(instrument_ids) + instrument_ids_to_decode: set[str] = {i.value for i in instrument_ids} dataset = self._check_all_datasets_equal(instrument_ids) - live_client = databento.Live(key=self._live_api_key, gateway=self._live_gateway) - - try: - live_client.subscribe( - dataset=dataset, - schema=databento.Schema.DEFINITION, - symbols=[i.symbol.value for i in instrument_ids], - stype_in=databento.SType.RAW_SYMBOL, - start=0, # From start of current session (latest definition) - ) - for record in live_client: - if isinstance(record, databento.SystemMsg) and record.is_heartbeat: - break - - if isinstance(record, databento.InstrumentDefMsg): - instrument = parse_record_with_metadata( - record, - publishers=self._loader.publishers, - ts_init=self._clock.timestamp_ns(), - ) - self.add(instrument=instrument) - self._log.debug(f"Added instrument {instrument.id}.") - - instrument_ids_to_decode.discard(instrument.id) - if not instrument_ids_to_decode: - break # All requested instrument IDs now decoded - finally: - # Close the connection (we will still process all received data) - live_client.stop() + live_client = nautilus_pyo3.DatabentoLiveClient( + key=self._live_api_key, + dataset=dataset, + publishers_path=str(PUBLISHERS_PATH), + ) + + pyo3_instruments = [] + + async def receive_instruments(pyo3_instrument) -> None: + print(f"Received {pyo3_instrument}") + pyo3_instruments.append(pyo3_instrument) + instrument_ids_to_decode.discard(instrument.id.value) + if not instrument_ids_to_decode: + raise asyncio.CancelledError("All instruments decoded") + + await live_client.subscribe( + schema="definition", + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), + start=0, # From start of current session (latest definition) + ) + + await asyncio.wait_for(live_client.start(callback=receive_instruments), timeout=5.0) + + instruments = instruments_from_pyo3(pyo3_instruments) + + for instrument in instruments: + self.add(instrument=instrument) + self._log.debug(f"Added instrument {instrument.id}.") async def load_async( self, diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 981d80c0ea1d..90b24f1cf193 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -326,6 +326,8 @@ class Bar: class BookOrder: ... class OrderBookDelta: + @property + def instrument_id(self) -> InstrumentId: ... @property def ts_event(self) -> int: ... @property @@ -1626,6 +1628,7 @@ class DatabentoDataLoader: self, path: PathLike[str] | str, ) -> None: ... + def schema_for_file(self, path: str) -> str: ... class DatabentoHistoricalClient: def __init__( @@ -1633,7 +1636,6 @@ class DatabentoHistoricalClient: key: str, publishers_path: str, ) -> None: ... - @property def key(self) -> str: ... async def get_dataset_range(self, dataset: str) -> dict[str, str]: ... @@ -1678,7 +1680,6 @@ class DatabentoLiveClient: dataset: str, publishers_path: str, ) -> None: ... - @property def key(self) -> str: ... @property diff --git a/poetry.lock b/poetry.lock index cd4d9f067f5f..e27715d383dc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -257,70 +257,6 @@ files = [ {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] -[[package]] -name = "cffi" -version = "1.16.0" -description = "Foreign Function Interface for Python calling C code." -optional = true -python-versions = ">=3.8" -files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, -] - -[package.dependencies] -pycparser = "*" - [[package]] name = "cfgv" version = "3.4.0" @@ -601,76 +537,6 @@ files = [ {file = "Cython-3.0.8.tar.gz", hash = "sha256:8333423d8fd5765e7cceea3a9985dd1e0a5dfeb2734629e1a2ed2d6233d39de6"}, ] -[[package]] -name = "databento" -version = "0.27.0" -description = "Official Python client library for Databento" -optional = true -python-versions = ">=3.8,<4.0" -files = [ - {file = "databento-0.27.0-py3-none-any.whl", hash = "sha256:f9f653e92141d768598e6c3a65a88b500eb0269f4f7f33915ad9e115794aab5e"}, - {file = "databento-0.27.0.tar.gz", hash = "sha256:26e5527d7431180fd68603a8a623848012901972b22e1875b3c96a26b988727a"}, -] - -[package.dependencies] -aiohttp = [ - {version = ">=3.8.3,<4.0.0", markers = "python_version < \"3.12\""}, - {version = ">=3.9.0,<4.0.0", markers = "python_version >= \"3.12\" and python_version < \"4.0\""}, -] -databento-dbn = "0.15.1" -numpy = [ - {version = ">=1.23.5", markers = "python_version < \"3.12\""}, - {version = ">=1.26.0,<2.0.0", markers = "python_version >= \"3.12\" and python_version < \"4.0\""}, -] -pandas = ">=1.5.3" -pyarrow = ">=13.0.0" -requests = ">=2.24.0" -zstandard = ">=0.21.0" - -[[package]] -name = "databento-dbn" -version = "0.15.1" -description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" -optional = true -python-versions = ">=3.8" -files = [ - {file = "databento_dbn-0.15.1-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:49142ac7f3bb7116636beeee874ab26e9cb48b6201e443960b57a40c9b509abf"}, - {file = "databento_dbn-0.15.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:735daefeac6e56d2a7073a92cec0c623e6104d9a102e67d4487e5693479c6895"}, - {file = "databento_dbn-0.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1949df559214a7a3f6982aaacde7ebbfe41f2da43e170c7b6c0ad6dbaf4c8559"}, - {file = "databento_dbn-0.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30c1dded9fd6d5af66fa87c9a9387e30ce91f2ecb98a325627bc82441e978d17"}, - {file = "databento_dbn-0.15.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2bcdd0331127d273a794846547aab744799a311e1e79f410cb03e39624f9f8fc"}, - {file = "databento_dbn-0.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d22008f25f460ad3b6fa8880e196c0313d5320dbbf524d1ac84bb3ca110fd623"}, - {file = "databento_dbn-0.15.1-cp310-none-win_amd64.whl", hash = "sha256:36270f476cbc7c3d025ac5f41350251e9e818d4ab4f6746be16e380f778f1f8c"}, - {file = "databento_dbn-0.15.1-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6858d3a8ce8f083c6f19cd416b4b3894899f6690368117fd61419b3deee0c51c"}, - {file = "databento_dbn-0.15.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:56ee6260da33cce64a95dfdb8b882502535f9bac85f6ec65c9c13d0744d516c8"}, - {file = "databento_dbn-0.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51c35086167311ecc7b873a48c5cd8106df676745a1d127a529b5bb25827ce74"}, - {file = "databento_dbn-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b79e1c7a03246cd5a071345bf1e2fe85d2d76a7d62cd22acdf0f4293bd76844f"}, - {file = "databento_dbn-0.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53f52b036d73ff08cf3810cbfb4bda824236ac3b88a401d1595167f2040a9e91"}, - {file = "databento_dbn-0.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:177b5edfeb9212d8fe9c69c5fc0b61aab441efb3afa843476b07ab026544256d"}, - {file = "databento_dbn-0.15.1-cp311-none-win_amd64.whl", hash = "sha256:1f32c1978fa013ccc453af194c60e5cef33058b695e1c57f42fa40d6b6bc0d4a"}, - {file = "databento_dbn-0.15.1-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:21f525ddde3ebd6be1aa564fa02c5961b6be2f7e53ab2eb37201bb1a0ae5b08b"}, - {file = "databento_dbn-0.15.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:83393da5261a7b2572552ff65c77048aee84f26fe5e9b41c314b395ae3c1e133"}, - {file = "databento_dbn-0.15.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6dcb293a7c90300afe2aa5b95d9d343bb0c327a36376d2635daed5ee8423a9e2"}, - {file = "databento_dbn-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f609fc2de3fbbfe0193427e58196b255307ad107b1e35d9edf6334998adadcb"}, - {file = "databento_dbn-0.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:385a9dcfe4876f8703edc975235c1bd31eaa7d1656e7c435ef25d489e9c81836"}, - {file = "databento_dbn-0.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8f937a97819301bca13d662aa29a7ec805b015c35ef875488e103121390be53"}, - {file = "databento_dbn-0.15.1-cp312-none-win_amd64.whl", hash = "sha256:d3941dd60062783cc67b52cf2a5bf672f688ce3b20ab2224a32ea6aa2cc5a808"}, - {file = "databento_dbn-0.15.1-cp38-cp38-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:f366914cd9315d44d44d8114048a6380184e21f1328345b6f8af47ff52914212"}, - {file = "databento_dbn-0.15.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d77727b2809b8950284b8e948bf5ee471c52b1587aa980ea5ba6061593362f68"}, - {file = "databento_dbn-0.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80ddd4ce7d2ef93cc87076a3d5639802ba06bd6ee5fe605001fb76d1eb7e25e0"}, - {file = "databento_dbn-0.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c6d54d5e95fa38dc412db4e41fe821fcb8ba309cf2f8660187982429ca936e8"}, - {file = "databento_dbn-0.15.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5951e70d740b57aa9bf1e7a9814119e73d4b3a3a3b768931baceafb5a70104c8"}, - {file = "databento_dbn-0.15.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:d8807f83a3cc3b2f7414f11980bf5df30b474a5b000fd01c981399e1547b063b"}, - {file = "databento_dbn-0.15.1-cp38-none-win_amd64.whl", hash = "sha256:39a0bedade76ff16198b7e332d769ee594b601e0a07129cbb0475090262a76db"}, - {file = "databento_dbn-0.15.1-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cffe7ae3da93dcb666b87557c27824fa54b28702bf9950358899fd5398980a1c"}, - {file = "databento_dbn-0.15.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:1107a06f243f0e2545ea86810ff55b7d29a91100a20bdcfd5db9ca9182986560"}, - {file = "databento_dbn-0.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a93bb51533c9b4eb7c349c4c93384feab32e22f07c7cb08a85986a1bdf6647b0"}, - {file = "databento_dbn-0.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:996e0d228e1fe31d879023d77b03717b4d7c26680f5b9e51bd0ceca01a967d14"}, - {file = "databento_dbn-0.15.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4bcbbbd34708905bc4131aea655c4d5a35b6bad14de151bdc110fb6da97739fb"}, - {file = "databento_dbn-0.15.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8811efb2b2a621bc1d210ee643287cf327779c181f34bb2d4cc4f29f8d97f7e4"}, - {file = "databento_dbn-0.15.1-cp39-none-win_amd64.whl", hash = "sha256:ea0315854ce3c43a9e4dd70b63f6a315c9a84381e6e47100cd2df4bbdfeec8b2"}, -] - [[package]] name = "defusedxml" version = "0.7.1" @@ -1759,17 +1625,6 @@ files = [ [package.dependencies] numpy = ">=1.16.6,<2" -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] - [[package]] name = "pygments" version = "2.17.2" @@ -2731,74 +2586,12 @@ files = [ idna = ">=2.0" multidict = ">=4.0" -[[package]] -name = "zstandard" -version = "0.22.0" -description = "Zstandard bindings for Python" -optional = true -python-versions = ">=3.8" -files = [ - {file = "zstandard-0.22.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:275df437ab03f8c033b8a2c181e51716c32d831082d93ce48002a5227ec93019"}, - {file = "zstandard-0.22.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ac9957bc6d2403c4772c890916bf181b2653640da98f32e04b96e4d6fb3252a"}, - {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe3390c538f12437b859d815040763abc728955a52ca6ff9c5d4ac707c4ad98e"}, - {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1958100b8a1cc3f27fa21071a55cb2ed32e9e5df4c3c6e661c193437f171cba2"}, - {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e1856c8313bc688d5df069e106a4bc962eef3d13372020cc6e3ebf5e045202"}, - {file = "zstandard-0.22.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1a90ba9a4c9c884bb876a14be2b1d216609385efb180393df40e5172e7ecf356"}, - {file = "zstandard-0.22.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3db41c5e49ef73641d5111554e1d1d3af106410a6c1fb52cf68912ba7a343a0d"}, - {file = "zstandard-0.22.0-cp310-cp310-win32.whl", hash = "sha256:d8593f8464fb64d58e8cb0b905b272d40184eac9a18d83cf8c10749c3eafcd7e"}, - {file = "zstandard-0.22.0-cp310-cp310-win_amd64.whl", hash = "sha256:f1a4b358947a65b94e2501ce3e078bbc929b039ede4679ddb0460829b12f7375"}, - {file = "zstandard-0.22.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:589402548251056878d2e7c8859286eb91bd841af117dbe4ab000e6450987e08"}, - {file = "zstandard-0.22.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a97079b955b00b732c6f280d5023e0eefe359045e8b83b08cf0333af9ec78f26"}, - {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:445b47bc32de69d990ad0f34da0e20f535914623d1e506e74d6bc5c9dc40bb09"}, - {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33591d59f4956c9812f8063eff2e2c0065bc02050837f152574069f5f9f17775"}, - {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:888196c9c8893a1e8ff5e89b8f894e7f4f0e64a5af4d8f3c410f0319128bb2f8"}, - {file = "zstandard-0.22.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:53866a9d8ab363271c9e80c7c2e9441814961d47f88c9bc3b248142c32141d94"}, - {file = "zstandard-0.22.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4ac59d5d6910b220141c1737b79d4a5aa9e57466e7469a012ed42ce2d3995e88"}, - {file = "zstandard-0.22.0-cp311-cp311-win32.whl", hash = "sha256:2b11ea433db22e720758cba584c9d661077121fcf60ab43351950ded20283440"}, - {file = "zstandard-0.22.0-cp311-cp311-win_amd64.whl", hash = "sha256:11f0d1aab9516a497137b41e3d3ed4bbf7b2ee2abc79e5c8b010ad286d7464bd"}, - {file = "zstandard-0.22.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6c25b8eb733d4e741246151d895dd0308137532737f337411160ff69ca24f93a"}, - {file = "zstandard-0.22.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9b2cde1cd1b2a10246dbc143ba49d942d14fb3d2b4bccf4618d475c65464912"}, - {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a88b7df61a292603e7cd662d92565d915796b094ffb3d206579aaebac6b85d5f"}, - {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466e6ad8caefb589ed281c076deb6f0cd330e8bc13c5035854ffb9c2014b118c"}, - {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1d67d0d53d2a138f9e29d8acdabe11310c185e36f0a848efa104d4e40b808e4"}, - {file = "zstandard-0.22.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:39b2853efc9403927f9065cc48c9980649462acbdf81cd4f0cb773af2fd734bc"}, - {file = "zstandard-0.22.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8a1b2effa96a5f019e72874969394edd393e2fbd6414a8208fea363a22803b45"}, - {file = "zstandard-0.22.0-cp312-cp312-win32.whl", hash = "sha256:88c5b4b47a8a138338a07fc94e2ba3b1535f69247670abfe422de4e0b344aae2"}, - {file = "zstandard-0.22.0-cp312-cp312-win_amd64.whl", hash = "sha256:de20a212ef3d00d609d0b22eb7cc798d5a69035e81839f549b538eff4105d01c"}, - {file = "zstandard-0.22.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d75f693bb4e92c335e0645e8845e553cd09dc91616412d1d4650da835b5449df"}, - {file = "zstandard-0.22.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:36a47636c3de227cd765e25a21dc5dace00539b82ddd99ee36abae38178eff9e"}, - {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68953dc84b244b053c0d5f137a21ae8287ecf51b20872eccf8eaac0302d3e3b0"}, - {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2612e9bb4977381184bb2463150336d0f7e014d6bb5d4a370f9a372d21916f69"}, - {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23d2b3c2b8e7e5a6cb7922f7c27d73a9a615f0a5ab5d0e03dd533c477de23004"}, - {file = "zstandard-0.22.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d43501f5f31e22baf822720d82b5547f8a08f5386a883b32584a185675c8fbf"}, - {file = "zstandard-0.22.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a493d470183ee620a3df1e6e55b3e4de8143c0ba1b16f3ded83208ea8ddfd91d"}, - {file = "zstandard-0.22.0-cp38-cp38-win32.whl", hash = "sha256:7034d381789f45576ec3f1fa0e15d741828146439228dc3f7c59856c5bcd3292"}, - {file = "zstandard-0.22.0-cp38-cp38-win_amd64.whl", hash = "sha256:d8fff0f0c1d8bc5d866762ae95bd99d53282337af1be9dc0d88506b340e74b73"}, - {file = "zstandard-0.22.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2fdd53b806786bd6112d97c1f1e7841e5e4daa06810ab4b284026a1a0e484c0b"}, - {file = "zstandard-0.22.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:73a1d6bd01961e9fd447162e137ed949c01bdb830dfca487c4a14e9742dccc93"}, - {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9501f36fac6b875c124243a379267d879262480bf85b1dbda61f5ad4d01b75a3"}, - {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48f260e4c7294ef275744210a4010f116048e0c95857befb7462e033f09442fe"}, - {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959665072bd60f45c5b6b5d711f15bdefc9849dd5da9fb6c873e35f5d34d8cfb"}, - {file = "zstandard-0.22.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d22fdef58976457c65e2796e6730a3ea4a254f3ba83777ecfc8592ff8d77d303"}, - {file = "zstandard-0.22.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a7ccf5825fd71d4542c8ab28d4d482aace885f5ebe4b40faaa290eed8e095a4c"}, - {file = "zstandard-0.22.0-cp39-cp39-win32.whl", hash = "sha256:f058a77ef0ece4e210bb0450e68408d4223f728b109764676e1a13537d056bb0"}, - {file = "zstandard-0.22.0-cp39-cp39-win_amd64.whl", hash = "sha256:e9e9d4e2e336c529d4c435baad846a181e39a982f823f7e4495ec0b0ec8538d2"}, - {file = "zstandard-0.22.0.tar.gz", hash = "sha256:8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70"}, -] - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - [extras] betfair = ["betfair_parser"] -databento = ["databento"] docker = ["docker"] ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "132915d961ce55fc8cd96ad424fade816e9a4e9a54bbd7cd2817424c1d21b706" +content-hash = "cd49b4783329bf0b0e1f170c3d92a37527b103b5e32f255e82ab4063f9261c64" diff --git a/pyproject.toml b/pyproject.toml index 9edb5e885690..e1073558272e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,14 +65,12 @@ uvloop = {version = "^0.19.0", markers = "sys_platform != 'win32'"} async-timeout = {version = "^4.0.3", optional = true} betfair_parser = {version = "==0.10.0", optional = true} # Pinned for stability -databento = {version = "^0.27.0", optional = true} defusedxml = {version = "^0.7.1", optional = true} docker = {version = "^6.1.3", optional = true} nautilus_ibapi = {version = "==10.19.2", optional = true} # Pinned for stability [tool.poetry.extras] betfair = ["betfair_parser"] -databento = ["databento"] docker = ["docker"] ib = ["nautilus_ibapi", "async-timeout", "defusedxml"] From 01c90d19c87fe168b6e8d5000fef4461ff89860f Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 12:30:03 +1100 Subject: [PATCH 14/42] Refine DatabentoLiveClient --- .../adapters/src/databento/python/live.rs | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 1c50f0169dff..bb714ac1447c 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -21,7 +21,7 @@ use anyhow::Result; use databento::live::Subscription; use dbn::{PitSymbolMap, RType, Record, SymbolIndex}; use indexmap::IndexMap; -use log::error; +use log::{error, info}; use nautilus_core::python::to_pyruntime_err; use nautilus_core::{ python::to_pyvalue_err, @@ -164,8 +164,8 @@ impl DatabentoLiveClient { continue; } RType::System => { - eprintln!("{:?}", record); // TODO: Just print stderr for now - error!("{:?}", record); + println!("{:?}", record); // TODO: Just print stderr for now + info!("{:?}", record); continue; } RType::InstrumentDef => { @@ -193,39 +193,40 @@ impl DatabentoLiveClient { } continue; } - _ => {} // Fall through - } + _ => { + let raw_symbol = symbol_map + .get_for_rec(&record) + .expect("Cannot resolve raw_symbol from `symbol_map`"); + + let symbol = Symbol::from_str_unchecked(raw_symbol); + let publisher_id = record.publisher().unwrap() as PublisherId; + let venue_str = publishers.get(&publisher_id).unwrap().venue.as_str(); + let venue = Venue::from_str_unchecked(venue_str); - let raw_symbol = symbol_map - .get_for_rec(&record) - .expect("Cannot resolve raw_symbol from `symbol_map`"); - - let symbol = Symbol::from_str_unchecked(raw_symbol); - let publisher_id = record.publisher().unwrap() as PublisherId; - let venue_str = publishers.get(&publisher_id).unwrap().venue.as_str(); - let venue = Venue::from_str_unchecked(venue_str); - - let instrument_id = InstrumentId::new(symbol, venue); - let ts_init = clock.get_time_ns(); - - let (data, _) = parse_record(&record, rtype, instrument_id, 2, Some(ts_init)) - .map_err(to_pyvalue_err)?; - - // TODO: Improve the efficiency of this constant GIL aquisition - Python::with_gil(|py| { - let py_obj = match data { - Data::Delta(delta) => delta.into_py(py), - Data::Depth10(depth) => depth.into_py(py), - Data::Quote(quote) => quote.into_py(py), - Data::Trade(trade) => trade.into_py(py), - _ => panic!("Invalid data element, was {:?}", data), - }; - - match callback.call1(py, (py_obj,)) { - Ok(_) => {} - Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now - }; - }) + let instrument_id = InstrumentId::new(symbol, venue); + let ts_init = clock.get_time_ns(); + + let (data, _) = + parse_record(&record, rtype, instrument_id, 2, Some(ts_init)) + .map_err(to_pyvalue_err)?; + + // TODO: Improve the efficiency of this constant GIL aquisition + Python::with_gil(|py| { + let py_obj = match data { + Data::Delta(delta) => delta.into_py(py), + Data::Depth10(depth) => depth.into_py(py), + Data::Quote(quote) => quote.into_py(py), + Data::Trade(trade) => trade.into_py(py), + _ => panic!("Invalid data element, was {:?}", data), + }; + + match callback.call1(py, (py_obj,)) { + Ok(_) => {} + Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now + }; + }) + } + } } Ok(()) }) From 717534b6025d48d93ac57bd7d79d9ef8076fd1b7 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 12:37:39 +1100 Subject: [PATCH 15/42] Add data pyo3 conversion methods --- nautilus_trader/adapters/databento/data.py | 14 +- nautilus_trader/core/nautilus_pyo3.pyi | 35 ++++- nautilus_trader/model/data.pxd | 12 ++ nautilus_trader/model/data.pyx | 133 +++++++++++++++++- nautilus_trader/test_kit/rust/data_pyo3.py | 33 ++++- tests/unit_tests/model/test_bar.py | 22 +++ tests/unit_tests/model/test_orderbook_data.py | 24 ++++ tests/unit_tests/model/test_tick.py | 43 ++++++ 8 files changed, 303 insertions(+), 13 deletions(-) diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index c3695b2de923..885a485fd692 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -811,9 +811,8 @@ def _handle_record( # self._log.debug(f"Received {record}", LogColor.MAGENTA) if isinstance(pyo3_data, nautilus_pyo3.OrderBookDelta): - instrument_id = InstrumentId.from_str(pyo3_data.instrument_id.value) - - data = OrderBookDelta.from_pyo3_list([pyo3_data]) # TODO: Implement single function + data = OrderBookDelta.from_pyo3(pyo3_data) + instrument_id = data.instrument_id if DatabentoRecordFlags.F_LAST not in DatabentoRecordFlags(data.flags): buffer = self._buffered_deltas[instrument_id] buffer.append(data) @@ -824,13 +823,14 @@ def _handle_record( data = OrderBookDeltas(instrument_id, deltas=buffer.copy()) buffer.clear() elif isinstance(pyo3_data, nautilus_pyo3.OrderBookDepth10): - data = OrderBookDepth10.from_pyo3_list([pyo3_data]) # TODO: Implement single function + raise RuntimeError("MBP-10 not currently supported: pyo3 conversion function needed") + data = OrderBookDepth10.from_pyo3(pyo3_data) elif isinstance(pyo3_data, nautilus_pyo3.QuoteTick): - data = QuoteTick.from_pyo3_list([pyo3_data]) # TODO: Implement single function + data = QuoteTick.from_pyo3(pyo3_data) elif isinstance(pyo3_data, nautilus_pyo3.TradeTick): - data = TradeTick.from_pyo3_list([pyo3_data]) # TODO: Implement single function + data = TradeTick.from_pyo3(pyo3_data) elif isinstance(pyo3_data, nautilus_pyo3.Bar): - data = Bar.from_pyo3_list([pyo3_data]) # TODO: Implement single function + data = Bar.from_pyo3(pyo3_data) else: self._log.error(f"Unimplemented data type in handler: {pyo3_data}.") return diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 90b24f1cf193..726700d72300 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -323,12 +323,45 @@ class Bar: @staticmethod def get_fields() -> dict[str, str]: ... -class BookOrder: ... +class BookOrder: + def __init__( + self, + side: OrderSide, + price: Price, + size: Quantity, + order_id: int, + ) -> None: ... + @property + def side(self) -> OrderSide: ... + @property + def price(self) -> Price: ... + @property + def size(self) -> Quantity: ... + @property + def order_id(self) -> int: ... class OrderBookDelta: + def __init__( + self, + instrument_id: InstrumentId, + action: BookAction, + order: BookOrder | None, + flags: int, + sequence: int, + ts_event: int, + ts_init: int, + ) -> None: ... @property def instrument_id(self) -> InstrumentId: ... @property + def action(self) -> BookAction: ... + @property + def order(self) -> BookOrder: ... + @property + def flags(self) -> int: ... + @property + def sequence(self) -> int: ... + @property def ts_event(self) -> int: ... @property def ts_init(self) -> int: ... diff --git a/nautilus_trader/model/data.pxd b/nautilus_trader/model/data.pxd index cc681a9b2c0b..69a5fa669523 100644 --- a/nautilus_trader/model/data.pxd +++ b/nautilus_trader/model/data.pxd @@ -152,6 +152,9 @@ cdef class Bar(Data): @staticmethod cdef Bar from_dict_c(dict values) + @staticmethod + cdef Bar from_pyo3_c(pyo3_bar) + @staticmethod cdef dict to_dict_c(Bar obj) @@ -209,6 +212,9 @@ cdef class OrderBookDelta(Data): @staticmethod cdef OrderBookDelta from_dict_c(dict values) + @staticmethod + cdef OrderBookDelta from_pyo3_c(pyo3_delta) + @staticmethod cdef dict to_dict_c(OrderBookDelta obj) @@ -356,6 +362,9 @@ cdef class QuoteTick(Data): @staticmethod cdef QuoteTick from_dict_c(dict values) + @staticmethod + cdef QuoteTick from_pyo3_c(pyo3_quote) + @staticmethod cdef dict to_dict_c(QuoteTick obj) @@ -393,6 +402,9 @@ cdef class TradeTick(Data): @staticmethod cdef TradeTick from_dict_c(dict values) + @staticmethod + cdef TradeTick from_pyo3_c(pyo3_trade) + @staticmethod cdef dict to_dict_c(TradeTick obj) diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 56849d50e3b9..0686ddc913af 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -874,6 +874,22 @@ cdef class Bar(Data): "ts_init": obj._mem.ts_init, } + @staticmethod + cdef Bar from_pyo3_c(pyo3_bar): + cdef uint8_t price_prec = pyo3_bar.open.precision + cdef uint8_t volume_prec = pyo3_bar.volume.precision + + return Bar( + BarType.from_str_c(str(pyo3_bar.bar_type)), + Price.from_raw_c(pyo3_bar.open.raw, price_prec), + Price.from_raw_c(pyo3_bar.high.raw, price_prec), + Price.from_raw_c(pyo3_bar.low.raw, price_prec), + Price.from_raw_c(pyo3_bar.close.raw, price_prec), + Quantity.from_raw_c(pyo3_bar.volume.raw, volume_prec), + pyo3_bar.ts_event, + pyo3_bar.ts_init, + ) + @property def bar_type(self) -> BarType: """ @@ -1087,6 +1103,23 @@ cdef class Bar(Data): return output + @staticmethod + def from_pyo3(pyo3_bar) -> Bar: + """ + Return a legacy Cython bar converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_bar : nautilus_pyo3.Bar + The pyo3 Rust bar to convert from. + + Returns + ------- + Bar + + """ + return Bar.from_pyo3_c(pyo3_bar) + cpdef bint is_single_price(self): """ If the OHLC are all equal to a single price. @@ -1783,6 +1816,23 @@ cdef class OrderBookDelta(Data): "ts_init": obj._mem.ts_init, } + @staticmethod + cdef OrderBookDelta from_pyo3_c(pyo3_delta): + return OrderBookDelta.from_raw_c( + InstrumentId.from_str_c(pyo3_delta.instrument_id.value), + pyo3_delta.action.value, + pyo3_delta.order.side.value, + pyo3_delta.order.price.raw, + pyo3_delta.order.price.precision, + pyo3_delta.order.size.raw, + pyo3_delta.order.size.precision, + pyo3_delta.order.order_id, + pyo3_delta.flags, + pyo3_delta.sequence, + pyo3_delta.ts_event, + pyo3_delta.ts_init, + ) + @staticmethod cdef OrderBookDelta clear_c( InstrumentId instrument_id, @@ -1997,6 +2047,23 @@ cdef class OrderBookDelta(Data): return output + @staticmethod + def from_pyo3(pyo3_delta) -> OrderBookDelta: + """ + Return a legacy Cython order book delta converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_delta : nautilus_pyo3.OrderBookDelta + The pyo3 Rust order book delta to convert from. + + Returns + ------- + OrderBookDelta + + """ + return OrderBookDelta.from_pyo3_c(pyo3_delta) + @staticmethod def from_pyo3_list(list pyo3_deltas) -> list[OrderBookDelta]: """ @@ -3121,6 +3188,22 @@ cdef class QuoteTick(Data): quote._mem = mem return quote + @staticmethod + cdef QuoteTick from_pyo3_c(pyo3_quote): + return QuoteTick.from_raw_c( + InstrumentId.from_str_c(pyo3_quote.instrument_id.value), + pyo3_quote.bid_price.raw, + pyo3_quote.ask_price.raw, + pyo3_quote.bid_price.precision, + pyo3_quote.ask_price.precision, + pyo3_quote.bid_size.raw, + pyo3_quote.ask_size.raw, + pyo3_quote.bid_size.precision, + pyo3_quote.ask_size.precision, + pyo3_quote.ts_event, + pyo3_quote.ts_init, + ) + # SAFETY: Do NOT deallocate the capsule here # It is supposed to be deallocated by the creator @staticmethod @@ -3359,6 +3442,23 @@ cdef class QuoteTick(Data): return output + @staticmethod + def from_pyo3(pyo3_quote) -> QuoteTick: + """ + Return a legacy Cython quote tick converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_quote : nautilus_pyo3.QuoteTick + The pyo3 Rust quote tick to convert from. + + Returns + ------- + QuoteTick + + """ + return QuoteTick.from_pyo3_c(pyo3_quote) + cpdef Price extract_price(self, PriceType price_type): """ Extract the price for the given price type. @@ -3685,6 +3785,20 @@ cdef class TradeTick(Data): "ts_init": obj.ts_init, } + @staticmethod + cdef TradeTick from_pyo3_c(pyo3_trade): + return TradeTick.from_raw_c( + InstrumentId.from_str_c(pyo3_trade.instrument_id.value), + pyo3_trade.price.raw, + pyo3_trade.price.precision, + pyo3_trade.size.raw, + pyo3_trade.size.precision, + pyo3_trade.aggressor_side.value, + TradeId(pyo3_trade.trade_id.value), + pyo3_trade.ts_event, + pyo3_trade.ts_init, + ) + @staticmethod def from_raw( InstrumentId instrument_id, @@ -3836,7 +3950,7 @@ cdef class TradeTick(Data): if instrument_id is None: instrument_id = InstrumentId.from_str_c(pyo3_trade.instrument_id.value) price_prec = pyo3_trade.price.precision - size_prec = pyo3_trade.price.precision + size_prec = pyo3_trade.size.precision trade = TradeTick.from_raw_c( instrument_id, @@ -3852,3 +3966,20 @@ cdef class TradeTick(Data): output.append(trade) return output + + @staticmethod + def from_pyo3(pyo3_trade) -> TradeTick: + """ + Return a legacy Cython trade tick converted from the given pyo3 Rust object. + + Parameters + ---------- + pyo3_trade : nautilus_pyo3.TradeTick + The pyo3 Rust trade tick to convert from. + + Returns + ------- + TradeTick + + """ + return TradeTick.from_pyo3_c(pyo3_trade) diff --git a/nautilus_trader/test_kit/rust/data_pyo3.py b/nautilus_trader/test_kit/rust/data_pyo3.py index 2e860c16fef8..024bed4fc0da 100644 --- a/nautilus_trader/test_kit/rust/data_pyo3.py +++ b/nautilus_trader/test_kit/rust/data_pyo3.py @@ -18,7 +18,11 @@ from nautilus_trader.core.nautilus_pyo3 import BarAggregation from nautilus_trader.core.nautilus_pyo3 import BarSpecification from nautilus_trader.core.nautilus_pyo3 import BarType +from nautilus_trader.core.nautilus_pyo3 import BookAction +from nautilus_trader.core.nautilus_pyo3 import BookOrder from nautilus_trader.core.nautilus_pyo3 import InstrumentId +from nautilus_trader.core.nautilus_pyo3 import OrderBookDelta +from nautilus_trader.core.nautilus_pyo3 import OrderSide from nautilus_trader.core.nautilus_pyo3 import Price from nautilus_trader.core.nautilus_pyo3 import PriceType from nautilus_trader.core.nautilus_pyo3 import Quantity @@ -28,6 +32,29 @@ class TestDataProviderPyo3: + @staticmethod + def order_book_delta( + instrument_id: InstrumentId | None = None, + price: float = 10000.0, + size: float = 0.1, + ts_event: int = 0, + ts_init: int = 0, + ) -> OrderBookDelta: + return OrderBookDelta( + instrument_id=instrument_id or TestIdProviderPyo3.ethusdt_binance_id(), + action=BookAction.ADD, + order=BookOrder( + side=OrderSide.BUY, + price=Price.from_str(str(price)), + size=Quantity.from_str(str(size)), + order_id=0, + ), + flags=0, + sequence=0, + ts_init=ts_init, + ts_event=ts_event, + ) + @staticmethod def trade_tick( instrument_id: InstrumentId | None = None, @@ -36,9 +63,8 @@ def trade_tick( ts_event: int = 0, ts_init: int = 0, ) -> TradeTick: - inst = instrument_id or TestIdProviderPyo3.ethusdt_binance_id() return TradeTick( - instrument_id=inst, + instrument_id=instrument_id or TestIdProviderPyo3.ethusdt_binance_id(), price=Price.from_str(str(price)), size=Quantity.from_str(str(size)), aggressor_side=AggressorSide.BUYER, @@ -57,9 +83,8 @@ def quote_tick( ts_event: int = 0, ts_init: int = 0, ) -> QuoteTick: - inst = instrument_id or TestIdProviderPyo3.ethusdt_binance_id() return QuoteTick( - instrument_id=inst, + instrument_id=instrument_id or TestIdProviderPyo3.ethusdt_binance_id(), bid_price=Price.from_str(str(bid_price)), ask_price=Price.from_str(str(ask_price)), bid_size=Quantity.from_str(str(bid_size)), diff --git a/tests/unit_tests/model/test_bar.py b/tests/unit_tests/model/test_bar.py index 866ed00572a9..f55512c13ceb 100644 --- a/tests/unit_tests/model/test_bar.py +++ b/tests/unit_tests/model/test_bar.py @@ -29,6 +29,7 @@ from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 from nautilus_trader.test_kit.stubs.data import TestDataStubs from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs @@ -609,6 +610,27 @@ def test_from_dict_returns_expected_bar(self): # Assert assert result == bar + def test_from_pyo3(self): + # Arrange + pyo3_bar = TestDataProviderPyo3.bar_5decimal() + + # Act + bar = Bar.from_pyo3(pyo3_bar) + + # Assert + assert isinstance(bar, Bar) + + def test_from_pyo3_list(self): + # Arrange + pyo3_bars = [TestDataProviderPyo3.bar_5decimal()] * 1024 + + # Act + bars = Bar.from_pyo3_list(pyo3_bars) + + # Assert + assert len(bars) == 1024 + assert isinstance(bars[0], Bar) + def test_pickle_bar(self): # Arrange bar = Bar( diff --git a/tests/unit_tests/model/test_orderbook_data.py b/tests/unit_tests/model/test_orderbook_data.py index 744105d69e8c..dd95870af774 100644 --- a/tests/unit_tests/model/test_orderbook_data.py +++ b/tests/unit_tests/model/test_orderbook_data.py @@ -26,6 +26,7 @@ from nautilus_trader.model.enums import OrderSide from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 from nautilus_trader.test_kit.stubs.data import TestDataStubs from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs @@ -555,6 +556,29 @@ def test_deltas_from_dict_returns_expected_dict() -> None: assert result == deltas +def test_deltas_from_pyo3(): + # Arrange + pyo3_delta = TestDataProviderPyo3.order_book_delta() + + # Act + delta = OrderBookDelta.from_pyo3(pyo3_delta) + + # Assert + assert isinstance(delta, OrderBookDelta) + + +def test_deltas_from_pyo3_list(): + # Arrange + pyo3_deltas = [TestDataProviderPyo3.order_book_delta()] * 1024 + + # Act + deltas = OrderBookDelta.from_pyo3_list(pyo3_deltas) + + # Assert + assert len(deltas) == 1024 + assert isinstance(deltas[0], OrderBookDelta) + + def test_depth10_fully_qualified_name() -> None: # Arrange, Act, Assert assert OrderBookDepth10.fully_qualified_name() == "nautilus_trader.model.data:OrderBookDepth10" diff --git a/tests/unit_tests/model/test_tick.py b/tests/unit_tests/model/test_tick.py index 041b29a52494..b840938cbd8f 100644 --- a/tests/unit_tests/model/test_tick.py +++ b/tests/unit_tests/model/test_tick.py @@ -26,6 +26,7 @@ from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity from nautilus_trader.test_kit.providers import TestInstrumentProvider +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 AUDUSD_SIM = TestInstrumentProvider.default_fx_ccy("AUD/USD") @@ -175,6 +176,27 @@ def test_from_raw_returns_expected_tick(self): assert tick.ts_event == 1 assert tick.ts_init == 2 + def test_from_pyo3(self): + # Arrange + pyo3_quote = TestDataProviderPyo3.quote_tick() + + # Act + quote = QuoteTick.from_pyo3(pyo3_quote) + + # Assert + assert isinstance(quote, QuoteTick) + + def test_from_pyo3_list(self): + # Arrange + pyo3_quotes = [TestDataProviderPyo3.quote_tick()] * 1024 + + # Act + quotes = QuoteTick.from_pyo3_list(pyo3_quotes) + + # Assert + assert len(quotes) == 1024 + assert isinstance(quotes[0], QuoteTick) + def test_pickling_round_trip_results_in_expected_tick(self): # Arrange tick = QuoteTick( @@ -262,6 +284,27 @@ def test_from_dict_returns_expected_tick(self): # Assert assert result == tick + def test_from_pyo3(self): + # Arrange + pyo3_trade = TestDataProviderPyo3.trade_tick() + + # Act + trade = TradeTick.from_pyo3(pyo3_trade) + + # Assert + assert isinstance(trade, TradeTick) + + def test_from_pyo3_list(self): + # Arrange + pyo3_trades = [TestDataProviderPyo3.trade_tick()] * 1024 + + # Act + trades = TradeTick.from_pyo3_list(pyo3_trades) + + # Assert + assert len(trades) == 1024 + assert isinstance(trades[0], TradeTick) + def test_pickling_round_trip_results_in_expected_tick(self): # Arrange tick = TradeTick( From 35d8f862af63b1662c9cbf1945c3532c5691047c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 14:40:03 +1100 Subject: [PATCH 16/42] Add cargo and clippy make targets --- Makefile | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f6716c4d7a4b..ca6837d06f76 100644 --- a/Makefile +++ b/Makefile @@ -76,13 +76,17 @@ docs-rust: clippy: (cd nautilus_core && cargo clippy --fix --all-targets --all-features -- -D warnings -W clippy::pedantic -W clippy::nursery -W clippy::unwrap_used -W clippy::expect_used) +.PHONY: clippy-nightly +clippy-nightly: + (cd nautilus_core && cargo +nightly clippy --fix --all-targets --all-features -- -D warnings -W clippy::pedantic -W clippy::nursery -W clippy::unwrap_used -W clippy::expect_used) + .PHONY: cargo-build cargo-build: (cd nautilus_core && cargo build --release --all-features) .PHONY: cargo-update cargo-update: - (cd nautilus_core && cargo update && cargo install cargo-nextest) + (cd nautilus_core && cargo update && cargo install cargo-nextest && cargo install cargo-llvm-cov) .PHONY: cargo-test cargo-test: @@ -92,9 +96,17 @@ cargo-test: fi RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace --exclude tokio-tungstenite) -.PHONY: cargo-test-nightly -cargo-test-nightly: - RUST_BACKTRACE=1 && (cd nautilus_core && cargo +nightly test) +.PHONY: cargo-test-coverage +cargo-test-coverage: + @if ! cargo nextest --version >/dev/null 2>&1; then \ + echo "cargo-nextest is not installed. You can install it using 'cargo install cargo-nextest'"; \ + exit 1; \ + fi + @if ! cargo llvm-cov --version >/dev/null 2>&1; then \ + echo "cargo-llvm-cov is not installed. You can install it using 'cargo install cargo-llvm-cov'"; \ + exit 1; \ + fi + RUST_BACKTRACE=1 && (cd nautilus_core && cargo llvm-cov nextest run --workspace --exclude tokio-tungstenite) .PHONY: cargo-bench cargo-bench: From 6ed2171d62a746db8f9bac7b1eba15362fc6a161 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 14:49:47 +1100 Subject: [PATCH 17/42] Fix clippy lints --- .pre-commit-config.yaml | 2 +- Makefile | 4 ---- .../adapters/src/databento/common.rs | 6 +++--- .../src/databento/python/historical.rs | 9 ++++----- .../adapters/src/databento/python/live.rs | 19 +++++++++---------- nautilus_core/rustfmt.toml | 2 +- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f59e0b308b69..a7b26014e491 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: hooks: - id: fmt name: cargo fmt - description: Format files with cargo fmt (nightly toolchain). + description: Format files with cargo fmt. entry: cargo fmt language: system types: [rust] diff --git a/Makefile b/Makefile index ca6837d06f76..572377c9d732 100644 --- a/Makefile +++ b/Makefile @@ -76,10 +76,6 @@ docs-rust: clippy: (cd nautilus_core && cargo clippy --fix --all-targets --all-features -- -D warnings -W clippy::pedantic -W clippy::nursery -W clippy::unwrap_used -W clippy::expect_used) -.PHONY: clippy-nightly -clippy-nightly: - (cd nautilus_core && cargo +nightly clippy --fix --all-targets --all-features -- -D warnings -W clippy::pedantic -W clippy::nursery -W clippy::unwrap_used -W clippy::expect_used) - .PHONY: cargo-build cargo-build: (cd nautilus_core && cargo build --release --all-features) diff --git a/nautilus_core/adapters/src/databento/common.rs b/nautilus_core/adapters/src/databento/common.rs index a866859859ef..49cd2cdbe703 100644 --- a/nautilus_core/adapters/src/databento/common.rs +++ b/nautilus_core/adapters/src/databento/common.rs @@ -41,11 +41,11 @@ pub fn nautilus_instrument_id_from_databento( pub fn get_date_time_range(start: UnixNanos, end: Option) -> Result { match end { Some(end) => Ok(DateTimeRange::from(( - OffsetDateTime::from_unix_timestamp_nanos(start as i128)?, - OffsetDateTime::from_unix_timestamp_nanos(end as i128)?, + OffsetDateTime::from_unix_timestamp_nanos(i128::from(start))?, + OffsetDateTime::from_unix_timestamp_nanos(i128::from(end))?, ))), None => Ok(DateTimeRange::from( - OffsetDateTime::from_unix_timestamp_nanos(start as i128)?, + OffsetDateTime::from_unix_timestamp_nanos(i128::from(start))?, )), } } diff --git a/nautilus_core/adapters/src/databento/python/historical.rs b/nautilus_core/adapters/src/databento/python/historical.rs index 526ec6ead0eb..f0a5a5415601 100644 --- a/nautilus_core/adapters/src/databento/python/historical.rs +++ b/nautilus_core/adapters/src/databento/python/historical.rs @@ -68,7 +68,6 @@ impl DatabentoHistoricalClient { let publishers_vec: Vec = serde_json::from_str(&file_content).map_err(to_pyvalue_err)?; let publishers = publishers_vec - .clone() .into_iter() .map(|p| (p.publisher_id, p)) .collect::>(); @@ -144,7 +143,7 @@ impl DatabentoHistoricalClient { let result = parse_instrument_def_msg(rec, publisher, ts_init); match result { Ok(instrument) => instruments.push(instrument), - Err(e) => eprintln!("{:?}", e), + Err(e) => eprintln!("{e:?}"), }; } @@ -214,7 +213,7 @@ impl DatabentoHistoricalClient { Data::Quote(quote) => { result.push(quote); } - _ => panic!("Invalid data element not `QuoteTick`, was {:?}", data), + _ => panic!("Invalid data element not `QuoteTick`, was {data:?}"), } } @@ -277,7 +276,7 @@ impl DatabentoHistoricalClient { Data::Trade(trade) => { result.push(trade); } - _ => panic!("Invalid data element not `TradeTick`, was {:?}", data), + _ => panic!("Invalid data element not `TradeTick`, was {data:?}"), } } @@ -349,7 +348,7 @@ impl DatabentoHistoricalClient { Data::Bar(bar) => { result.push(bar); } - _ => panic!("Invalid data element not `Bar`, was {:?}", data), + _ => panic!("Invalid data element not `Bar`, was {data:?}"), } } diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index bb714ac1447c..2b41b650e59e 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -83,7 +83,6 @@ impl DatabentoLiveClient { let publishers_vec: Vec = serde_json::from_str(&file_content).map_err(to_pyvalue_err)?; let publishers = publishers_vec - .clone() .into_iter() .map(|p| (p.publisher_id, p)) .collect::>(); @@ -120,7 +119,7 @@ impl DatabentoLiveClient { .schema(dbn::Schema::from_str(&schema).map_err(to_pyvalue_err)?) .stype_in(dbn::SType::from_str(&stype_in).map_err(to_pyvalue_err)?) .start( - OffsetDateTime::from_unix_timestamp_nanos(start as i128) + OffsetDateTime::from_unix_timestamp_nanos(i128::from(start)) .map_err(to_pyvalue_err)?, ) .build(), @@ -154,17 +153,17 @@ impl DatabentoLiveClient { match rtype { RType::SymbolMapping => { symbol_map.on_record(record).unwrap_or_else(|_| { - panic!("Error updating `symbol_map` with {:?}", record) + panic!("Error updating `symbol_map` with {record:?}") }); continue; } RType::Error => { - eprintln!("{:?}", record); // TODO: Just print stderr for now + eprintln!("{record:?}"); // TODO: Just print stderr for now error!("{:?}", record); continue; } RType::System => { - println!("{:?}", record); // TODO: Just print stderr for now + println!("{record:?}"); // TODO: Just print stderr for now info!("{:?}", record); continue; } @@ -185,11 +184,11 @@ impl DatabentoLiveClient { convert_instrument_to_pyobject(py, instrument).unwrap(); match callback.call1(py, (py_obj,)) { Ok(_) => {} - Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now + Err(e) => eprintln!("Error on callback, {e:?}"), // Just print error for now }; }); } - Err(e) => eprintln!("{:?}", e), + Err(e) => eprintln!("{e:?}"), } continue; } @@ -217,14 +216,14 @@ impl DatabentoLiveClient { Data::Depth10(depth) => depth.into_py(py), Data::Quote(quote) => quote.into_py(py), Data::Trade(trade) => trade.into_py(py), - _ => panic!("Invalid data element, was {:?}", data), + _ => panic!("Invalid data element, was {data:?}"), }; match callback.call1(py, (py_obj,)) { Ok(_) => {} - Err(e) => eprintln!("Error on callback, {:?}", e), // Just print error for now + Err(e) => eprintln!("Error on callback, {e:?}"), // Just print error for now }; - }) + }); } } } diff --git a/nautilus_core/rustfmt.toml b/nautilus_core/rustfmt.toml index 16f7ddf4c87a..8045ae5687f0 100644 --- a/nautilus_core/rustfmt.toml +++ b/nautilus_core/rustfmt.toml @@ -1,5 +1,5 @@ # This project uses rustfmt to format source code. Run `cargo +nightly fmt [-- --check]`. # https://github.com/rust-lang/rustfmt/blob/master/Configurations.md -group_imports = "StdExternalCrate" +# group_imports = "StdExternalCrate" imports_granularity = "Crate" From a2390af31dc991c21012cabcb852de3af57597a8 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 15:11:46 +1100 Subject: [PATCH 18/42] Fix datetime deprecation warning on build --- build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index c1bfd36a1733..cd49a4b62e0f 100644 --- a/build.py +++ b/build.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 +import datetime import itertools import os import platform import shutil import subprocess import sysconfig -from datetime import datetime from pathlib import Path import numpy as np +import pytz import toml from Cython.Build import build_ext from Cython.Build import cythonize @@ -350,7 +351,7 @@ def build() -> None: print(f"PYO3_ONLY={PYO3_ONLY}\n") print("Starting build...") - ts_start = datetime.utcnow() + ts_start = datetime.datetime.now(pytz.utc) build() - print(f"Build time: {datetime.utcnow() - ts_start}") + print(f"Build time: {datetime.datetime.now(pytz.utc) - ts_start}") print("\033[32m" + "Build completed" + "\033[0m") From 40c3a24f618c848d7263844105cda74a1ae12445 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 16:14:52 +1100 Subject: [PATCH 19/42] Fix build --- build.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index cd49a4b62e0f..4d453dd4a974 100644 --- a/build.py +++ b/build.py @@ -10,7 +10,6 @@ from pathlib import Path import numpy as np -import pytz import toml from Cython.Build import build_ext from Cython.Build import cythonize @@ -351,7 +350,7 @@ def build() -> None: print(f"PYO3_ONLY={PYO3_ONLY}\n") print("Starting build...") - ts_start = datetime.datetime.now(pytz.utc) + ts_start = datetime.datetime.now(datetime.timezone.utc) build() - print(f"Build time: {datetime.datetime.now(pytz.utc) - ts_start}") + print(f"Build time: {datetime.datetime.now(datetime.timezone.utc) - ts_start}") print("\033[32m" + "Build completed" + "\033[0m") From 9faa454426199524558691cb1cef621412a6515c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 21:59:25 +1100 Subject: [PATCH 20/42] Standardize property ordering --- nautilus_trader/model/data.pyx | 296 +++++++++++++++++---------------- 1 file changed, 155 insertions(+), 141 deletions(-) diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 0686ddc913af..d0917d754fde 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -164,6 +164,7 @@ cdef class BarSpecification: ------ ValueError If `step` is not positive (> 0). + """ def __init__( @@ -221,72 +222,6 @@ cdef class BarSpecification: def __repr__(self) -> str: return f"{type(self).__name__}({self})" - cdef str aggregation_string_c(self): - return bar_aggregation_to_str(self.aggregation) - - @staticmethod - cdef BarSpecification from_mem_c(BarSpecification_t mem): - cdef BarSpecification spec = BarSpecification.__new__(BarSpecification) - spec._mem = mem - return spec - - @staticmethod - cdef BarSpecification from_str_c(str value): - Condition.valid_string(value, 'value') - - cdef list pieces = value.rsplit('-', maxsplit=2) - - if len(pieces) != 3: - raise ValueError( - f"The `BarSpecification` string value was malformed, was {value}", - ) - - return BarSpecification( - int(pieces[0]), - bar_aggregation_from_str(pieces[1]), - price_type_from_str(pieces[2]), - ) - - @staticmethod - cdef bint check_time_aggregated_c(BarAggregation aggregation): - if ( - aggregation == BarAggregation.MILLISECOND - or aggregation == BarAggregation.SECOND - or aggregation == BarAggregation.MINUTE - or aggregation == BarAggregation.HOUR - or aggregation == BarAggregation.DAY - or aggregation == BarAggregation.WEEK - or aggregation == BarAggregation.MONTH - ): - return True - else: - return False - - @staticmethod - cdef bint check_threshold_aggregated_c(BarAggregation aggregation): - if ( - aggregation == BarAggregation.TICK - or aggregation == BarAggregation.TICK_IMBALANCE - or aggregation == BarAggregation.VOLUME - or aggregation == BarAggregation.VOLUME_IMBALANCE - or aggregation == BarAggregation.VALUE - or aggregation == BarAggregation.VALUE_IMBALANCE - ): - return True - else: - return False - - @staticmethod - cdef bint check_information_aggregated_c(BarAggregation aggregation): - if ( - aggregation == BarAggregation.TICK_RUNS - or aggregation == BarAggregation.VOLUME_RUNS - or aggregation == BarAggregation.VALUE_RUNS - ): - return True - else: - return False - @property def step(self) -> int: """ @@ -356,6 +291,73 @@ cdef class BarSpecification: f"{bar_aggregation_to_str(self.aggregation)}", ) + cdef str aggregation_string_c(self): + return bar_aggregation_to_str(self.aggregation) + + @staticmethod + cdef BarSpecification from_mem_c(BarSpecification_t mem): + cdef BarSpecification spec = BarSpecification.__new__(BarSpecification) + spec._mem = mem + return spec + + @staticmethod + cdef BarSpecification from_str_c(str value): + Condition.valid_string(value, 'value') + + cdef list pieces = value.rsplit('-', maxsplit=2) + + if len(pieces) != 3: + raise ValueError( + f"The `BarSpecification` string value was malformed, was {value}", + ) + + return BarSpecification( + int(pieces[0]), + bar_aggregation_from_str(pieces[1]), + price_type_from_str(pieces[2]), + ) + + @staticmethod + cdef bint check_time_aggregated_c(BarAggregation aggregation): + if ( + aggregation == BarAggregation.MILLISECOND + or aggregation == BarAggregation.SECOND + or aggregation == BarAggregation.MINUTE + or aggregation == BarAggregation.HOUR + or aggregation == BarAggregation.DAY + or aggregation == BarAggregation.WEEK + or aggregation == BarAggregation.MONTH + ): + return True + else: + return False + + @staticmethod + cdef bint check_threshold_aggregated_c(BarAggregation aggregation): + if ( + aggregation == BarAggregation.TICK + or aggregation == BarAggregation.TICK_IMBALANCE + or aggregation == BarAggregation.VOLUME + or aggregation == BarAggregation.VOLUME_IMBALANCE + or aggregation == BarAggregation.VALUE + or aggregation == BarAggregation.VALUE_IMBALANCE + ): + return True + else: + return False + + @staticmethod + cdef bint check_information_aggregated_c(BarAggregation aggregation): + if ( + aggregation == BarAggregation.TICK_RUNS + or aggregation == BarAggregation.VOLUME_RUNS + or aggregation == BarAggregation.VALUE_RUNS + ): + return True + else: + return False + + @staticmethod def from_str(str value) -> BarSpecification: """ @@ -557,6 +559,7 @@ cdef class BarType: ----- It is expected that all bar aggregation methods other than time will be internally aggregated. + """ def __init__( @@ -619,24 +622,6 @@ cdef class BarType: def __repr__(self) -> str: return f"{type(self).__name__}({self})" - @staticmethod - cdef BarType from_mem_c(BarType_t mem): - cdef BarType bar_type = BarType.__new__(BarType) - bar_type._mem = mem - return bar_type - - @staticmethod - cdef BarType from_str_c(str value): - Condition.valid_string(value, "value") - - cdef str parse_err = cstr_to_pystr(bar_type_check_parsing(pystr_to_cstr(value))) - if parse_err: - raise ValueError(parse_err) - - cdef BarType bar_type = BarType.__new__(BarType) - bar_type._mem = bar_type_from_cstr(pystr_to_cstr(value)) - return bar_type - @property def instrument_id(self) -> InstrumentId: """ @@ -673,6 +658,24 @@ cdef class BarType: """ return self._mem.aggregation_source + @staticmethod + cdef BarType from_mem_c(BarType_t mem): + cdef BarType bar_type = BarType.__new__(BarType) + bar_type._mem = mem + return bar_type + + @staticmethod + cdef BarType from_str_c(str value): + Condition.valid_string(value, "value") + + cdef str parse_err = cstr_to_pystr(bar_type_check_parsing(pystr_to_cstr(value))) + if parse_err: + raise ValueError(parse_err) + + cdef BarType bar_type = BarType.__new__(BarType) + bar_type._mem = bar_type_from_cstr(pystr_to_cstr(value)) + return bar_type + @staticmethod def from_str(str value) -> BarType: """ @@ -751,6 +754,7 @@ cdef class Bar(Data): If `high` is not >= `close`. ValueError If `low` is not <= `close`. + """ def __init__( @@ -839,57 +843,6 @@ cdef class Bar(Data): def __repr__(self) -> str: return f"{type(self).__name__}({self})" - @staticmethod - cdef Bar from_mem_c(Bar_t mem): - cdef Bar bar = Bar.__new__(Bar) - bar._mem = mem - return bar - - @staticmethod - cdef Bar from_dict_c(dict values): - Condition.not_none(values, "values") - return Bar( - bar_type=BarType.from_str_c(values["bar_type"]), - open=Price.from_str_c(values["open"]), - high=Price.from_str_c(values["high"]), - low=Price.from_str_c(values["low"]), - close=Price.from_str_c(values["close"]), - volume=Quantity.from_str_c(values["volume"]), - ts_event=values["ts_event"], - ts_init=values["ts_init"], - ) - - @staticmethod - cdef dict to_dict_c(Bar obj): - Condition.not_none(obj, "obj") - return { - "type": type(obj).__name__, - "bar_type": str(obj.bar_type), - "open": str(obj.open), - "high": str(obj.high), - "low": str(obj.low), - "close": str(obj.close), - "volume": str(obj.volume), - "ts_event": obj._mem.ts_event, - "ts_init": obj._mem.ts_init, - } - - @staticmethod - cdef Bar from_pyo3_c(pyo3_bar): - cdef uint8_t price_prec = pyo3_bar.open.precision - cdef uint8_t volume_prec = pyo3_bar.volume.precision - - return Bar( - BarType.from_str_c(str(pyo3_bar.bar_type)), - Price.from_raw_c(pyo3_bar.open.raw, price_prec), - Price.from_raw_c(pyo3_bar.high.raw, price_prec), - Price.from_raw_c(pyo3_bar.low.raw, price_prec), - Price.from_raw_c(pyo3_bar.close.raw, price_prec), - Quantity.from_raw_c(pyo3_bar.volume.raw, volume_prec), - pyo3_bar.ts_event, - pyo3_bar.ts_init, - ) - @property def bar_type(self) -> BarType: """ @@ -986,6 +939,57 @@ cdef class Bar(Data): """ return self._mem.ts_init + @staticmethod + cdef Bar from_mem_c(Bar_t mem): + cdef Bar bar = Bar.__new__(Bar) + bar._mem = mem + return bar + + @staticmethod + cdef Bar from_dict_c(dict values): + Condition.not_none(values, "values") + return Bar( + bar_type=BarType.from_str_c(values["bar_type"]), + open=Price.from_str_c(values["open"]), + high=Price.from_str_c(values["high"]), + low=Price.from_str_c(values["low"]), + close=Price.from_str_c(values["close"]), + volume=Quantity.from_str_c(values["volume"]), + ts_event=values["ts_event"], + ts_init=values["ts_init"], + ) + + @staticmethod + cdef dict to_dict_c(Bar obj): + Condition.not_none(obj, "obj") + return { + "type": type(obj).__name__, + "bar_type": str(obj.bar_type), + "open": str(obj.open), + "high": str(obj.high), + "low": str(obj.low), + "close": str(obj.close), + "volume": str(obj.volume), + "ts_event": obj._mem.ts_event, + "ts_init": obj._mem.ts_init, + } + + @staticmethod + cdef Bar from_pyo3_c(pyo3_bar): + cdef uint8_t price_prec = pyo3_bar.open.precision + cdef uint8_t volume_prec = pyo3_bar.volume.precision + + return Bar( + BarType.from_str_c(str(pyo3_bar.bar_type)), + Price.from_raw_c(pyo3_bar.open.raw, price_prec), + Price.from_raw_c(pyo3_bar.high.raw, price_prec), + Price.from_raw_c(pyo3_bar.low.raw, price_prec), + Price.from_raw_c(pyo3_bar.close.raw, price_prec), + Quantity.from_raw_c(pyo3_bar.volume.raw, volume_prec), + pyo3_bar.ts_event, + pyo3_bar.ts_init, + ) + @staticmethod def from_dict(dict values) -> Bar: """ @@ -1154,6 +1158,7 @@ cdef class DataType: -------- This class may be used as a key in hash maps throughout the system, thus the key and value contents of metadata must themselves be hashable. + """ def __init__(self, type type not None, dict metadata = None) -> None: # noqa (shadows built-in type) @@ -1204,6 +1209,7 @@ cdef class CustomData(Data): The data type. data : Data The data object to wrap. + """ def __init__( @@ -1265,6 +1271,7 @@ cdef class BookOrder: The order size. order_id : uint64_t The order ID. + """ def __init__( @@ -1332,12 +1339,6 @@ cdef class BookOrder: ) return order - @staticmethod - cdef BookOrder from_mem_c(BookOrder_t mem): - cdef BookOrder order = BookOrder.__new__(BookOrder) - order._mem = mem - return order - @property def price(self) -> Price: """ @@ -1386,6 +1387,12 @@ cdef class BookOrder: """ return self._mem.order_id + @staticmethod + cdef BookOrder from_mem_c(BookOrder_t mem): + cdef BookOrder order = BookOrder.__new__(BookOrder) + order._mem = mem + return order + cpdef double exposure(self): """ Return the total exposure for this order (price * size). @@ -1520,6 +1527,7 @@ cdef class OrderBookDelta(Data): A combination of packet end with matching engine status. sequence : uint64_t, default 0 The unique sequence number for the update. + """ def __init__( @@ -2128,6 +2136,7 @@ cdef class OrderBookDeltas(Data): ------ ValueError If `deltas` is an empty list. + """ def __init__( @@ -2247,6 +2256,7 @@ cdef class OrderBookDepth10(Data): If `bid_counts` length is not equal to 10. ValueError If `ask_counts` length is not equal to 10. + """ def __init__( @@ -2644,6 +2654,7 @@ cdef class VenueStatus(Data): The UNIX timestamp (nanoseconds) when the status update event occurred. ts_init : uint64_t The UNIX timestamp (nanoseconds) when the object was initialized. + """ def __init__( @@ -2854,6 +2865,7 @@ cdef class InstrumentClose(Data): The UNIX timestamp (nanoseconds) when the close price event occurred. ts_init : uint64_t The UNIX timestamp (nanoseconds) when the object was initialized. + """ def __init__( @@ -2966,6 +2978,7 @@ cdef class QuoteTick(Data): If `bid.precision` != `ask.precision`. ValueError If `bid_size.precision` != `ask_size.precision`. + """ def __init__( @@ -3534,6 +3547,7 @@ cdef class TradeTick(Data): ------ ValueError If `trade_id` is not a valid string. + """ def __init__( From db44984f496d565cd9bbcee9f5ce4649f1a1725f Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 22:17:50 +1100 Subject: [PATCH 21/42] Minor formatting --- nautilus_trader/core/nautilus_pyo3.pyi | 3 --- 1 file changed, 3 deletions(-) diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 726700d72300..50389f6268b3 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1541,7 +1541,6 @@ class SimpleMovingAverage: def has_inputs(self) -> bool: ... @property def value(self) -> float: ... - def update_raw(self, value: float) -> None: ... def reset(self) -> None: ... def handle_quote_tick(self, tick: QuoteTick) -> None: ... @@ -1616,7 +1615,6 @@ class HullMovingAverage: def has_inputs(self) -> bool: ... @property def value(self) -> float: ... - def update_raw(self, value: float) -> None: ... def handle_quote_tick(self, tick: QuoteTick) -> None: ... def handle_trade_tick(self, tick: TradeTick) -> None: ... @@ -1643,7 +1641,6 @@ class WilderMovingAverage: def value(self) -> float: ... @property def alpha(self) -> float: ... - def update_raw(self, value: float) -> None: ... def handle_quote_tick(self, tick: QuoteTick) -> None: ... def handle_trade_tick(self, tick: TradeTick) -> None: ... From b4c7c6be4fd21f02cd3a20e90b2ca77032004f26 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 22:32:58 +1100 Subject: [PATCH 22/42] Use pyo3 getters for data types --- nautilus_core/model/src/data/bar.rs | 14 ++++++ nautilus_core/model/src/data/delta.rs | 7 +++ nautilus_core/model/src/data/deltas.rs | 6 +++ nautilus_core/model/src/data/depth.rs | 9 ++++ nautilus_core/model/src/data/order.rs | 4 ++ nautilus_core/model/src/data/quote.rs | 7 +++ nautilus_core/model/src/data/trade.rs | 7 +++ nautilus_core/model/src/python/data/bar.rs | 40 ----------------- nautilus_core/model/src/python/data/delta.rs | 35 --------------- nautilus_core/model/src/python/data/depth.rs | 45 -------------------- nautilus_core/model/src/python/data/order.rs | 20 --------- nautilus_core/model/src/python/data/quote.rs | 35 --------------- nautilus_core/model/src/python/data/trade.rs | 35 --------------- 13 files changed, 54 insertions(+), 210 deletions(-) diff --git a/nautilus_core/model/src/data/bar.rs b/nautilus_core/model/src/data/bar.rs index bf021dd2b0ca..0d13a1b6ad3e 100644 --- a/nautilus_core/model/src/data/bar.rs +++ b/nautilus_core/model/src/data/bar.rs @@ -43,10 +43,13 @@ use crate::{ #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct BarSpecification { /// The step for binning samples for bar aggregation. + #[pyo3(get)] pub step: usize, /// The type of bar aggregation. + #[pyo3(get)] pub aggregation: BarAggregation, /// The price type to use for aggregation. + #[pyo3(get)] pub price_type: PriceType, } @@ -77,10 +80,13 @@ impl Display for BarSpecification { )] pub struct BarType { /// The bar types instrument ID. + #[pyo3(get)] pub instrument_id: InstrumentId, /// The bar types specification. + #[pyo3(get)] pub spec: BarSpecification, /// The bar types aggregation source. + #[pyo3(get)] pub aggregation_source: AggregationSource, } @@ -210,20 +216,28 @@ impl<'de> Deserialize<'de> for BarType { )] pub struct Bar { /// The bar type for this bar. + #[pyo3(get)] pub bar_type: BarType, /// The bars open price. + #[pyo3(get)] pub open: Price, /// The bars high price. + #[pyo3(get)] pub high: Price, /// The bars low price. + #[pyo3(get)] pub low: Price, /// The bars close price. + #[pyo3(get)] pub close: Price, /// The bars volume. + #[pyo3(get)] pub volume: Quantity, /// The UNIX timestamp (nanoseconds) when the data event occurred. + #[pyo3(get)] pub ts_event: UnixNanos, /// The UNIX timestamp (nanoseconds) when the data object was initialized. + #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/data/delta.rs b/nautilus_core/model/src/data/delta.rs index 7bf17b21e628..a85a91bae8af 100644 --- a/nautilus_core/model/src/data/delta.rs +++ b/nautilus_core/model/src/data/delta.rs @@ -43,18 +43,25 @@ use crate::{ #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct OrderBookDelta { /// The instrument ID for the book. + #[pyo3(get)] pub instrument_id: InstrumentId, /// The order book delta action. + #[pyo3(get)] pub action: BookAction, /// The order to apply. + #[pyo3(get)] pub order: BookOrder, /// A combination of packet end with matching engine status. + #[pyo3(get)] pub flags: u8, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u64, /// The UNIX timestamp (nanoseconds) when the data event occurred. + #[pyo3(get)] pub ts_event: UnixNanos, /// The UNIX timestamp (nanoseconds) when the data object was initialized. + #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/data/deltas.rs b/nautilus_core/model/src/data/deltas.rs index c6a66745be22..370e6addc082 100644 --- a/nautilus_core/model/src/data/deltas.rs +++ b/nautilus_core/model/src/data/deltas.rs @@ -30,16 +30,22 @@ use crate::identifiers::instrument_id::InstrumentId; )] pub struct OrderBookDeltas { /// The instrument ID for the book. + #[pyo3(get)] pub instrument_id: InstrumentId, /// The order book deltas. + #[pyo3(get)] pub deltas: Vec, /// A combination of packet end with matching engine status. + #[pyo3(get)] pub flags: u8, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u64, /// The UNIX timestamp (nanoseconds) when the data event occurred. + #[pyo3(get)] pub ts_event: UnixNanos, /// The UNIX timestamp (nanoseconds) when the data object was initialized. + #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/data/depth.rs b/nautilus_core/model/src/data/depth.rs index 4183359d89b3..012aeb23065c 100644 --- a/nautilus_core/model/src/data/depth.rs +++ b/nautilus_core/model/src/data/depth.rs @@ -46,22 +46,31 @@ pub const DEPTH10_LEN: usize = 10; #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct OrderBookDepth10 { /// The instrument ID for the book. + #[pyo3(get)] pub instrument_id: InstrumentId, /// The bid orders for the depth update. + #[pyo3(get)] pub bids: [BookOrder; DEPTH10_LEN], /// The ask orders for the depth update. + #[pyo3(get)] pub asks: [BookOrder; DEPTH10_LEN], /// The count of bid orders per level for the depth update. + #[pyo3(get)] pub bid_counts: [u32; DEPTH10_LEN], /// The count of ask orders per level for the depth update. + #[pyo3(get)] pub ask_counts: [u32; DEPTH10_LEN], /// A combination of packet end with matching engine status. + #[pyo3(get)] pub flags: u8, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u64, /// The UNIX timestamp (nanoseconds) when the data event occurred. + #[pyo3(get)] pub ts_event: UnixNanos, /// The UNIX timestamp (nanoseconds) when the data object was initialized. + #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/data/order.rs b/nautilus_core/model/src/data/order.rs index b2fa5b313b45..c826b9e2d793 100644 --- a/nautilus_core/model/src/data/order.rs +++ b/nautilus_core/model/src/data/order.rs @@ -54,12 +54,16 @@ pub const NULL_ORDER: BookOrder = BookOrder { #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct BookOrder { /// The order side. + #[pyo3(get)] pub side: OrderSide, /// The order price. + #[pyo3(get)] pub price: Price, /// The order size. + #[pyo3(get)] pub size: Quantity, /// The order ID. + #[pyo3(get)] pub order_id: OrderId, } diff --git a/nautilus_core/model/src/data/quote.rs b/nautilus_core/model/src/data/quote.rs index 5c6a6ec29364..f3621a56624a 100644 --- a/nautilus_core/model/src/data/quote.rs +++ b/nautilus_core/model/src/data/quote.rs @@ -47,18 +47,25 @@ use crate::{ #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct QuoteTick { /// The quotes instrument ID. + #[pyo3(get)] pub instrument_id: InstrumentId, /// The top of book bid price. + #[pyo3(get)] pub bid_price: Price, /// The top of book ask price. + #[pyo3(get)] pub ask_price: Price, /// The top of book bid size. + #[pyo3(get)] pub bid_size: Quantity, /// The top of book ask size. + #[pyo3(get)] pub ask_size: Quantity, /// The UNIX timestamp (nanoseconds) when the tick event occurred. + #[pyo3(get)] pub ts_event: UnixNanos, /// The UNIX timestamp (nanoseconds) when the data object was initialized. + #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/data/trade.rs b/nautilus_core/model/src/data/trade.rs index 504bdd62333c..3968912f39c9 100644 --- a/nautilus_core/model/src/data/trade.rs +++ b/nautilus_core/model/src/data/trade.rs @@ -42,18 +42,25 @@ use crate::{ #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct TradeTick { /// The trade instrument ID. + #[pyo3(get)] pub instrument_id: InstrumentId, /// The traded price. + #[pyo3(get)] pub price: Price, /// The traded size. + #[pyo3(get)] pub size: Quantity, /// The trade aggressor side. + #[pyo3(get)] pub aggressor_side: AggressorSide, /// The trade match ID (assigned by the venue). + #[pyo3(get)] pub trade_id: TradeId, /// The UNIX timestamp (nanoseconds) when the tick event occurred. + #[pyo3(get)] pub ts_event: UnixNanos, /// The UNIX timestamp (nanoseconds) when the data object was initialized. + #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/python/data/bar.rs b/nautilus_core/model/src/python/data/bar.rs index 7570f4d69c6e..600302c25946 100644 --- a/nautilus_core/model/src/python/data/bar.rs +++ b/nautilus_core/model/src/python/data/bar.rs @@ -164,46 +164,6 @@ impl Bar { format!("{self:?}") } - #[getter] - fn bar_type(&self) -> BarType { - self.bar_type - } - - #[getter] - fn open(&self) -> Price { - self.open - } - - #[getter] - fn high(&self) -> Price { - self.high - } - - #[getter] - fn low(&self) -> Price { - self.low - } - - #[getter] - fn close(&self) -> Price { - self.close - } - - #[getter] - fn volume(&self) -> Quantity { - self.volume - } - - #[getter] - fn ts_event(&self) -> UnixNanos { - self.ts_event - } - - #[getter] - fn ts_init(&self) -> UnixNanos { - self.ts_init - } - #[staticmethod] #[pyo3(name = "fully_qualified_name")] fn py_fully_qualified_name() -> String { diff --git a/nautilus_core/model/src/python/data/delta.rs b/nautilus_core/model/src/python/data/delta.rs index 47fc63ef6f30..0bc60470ef49 100644 --- a/nautilus_core/model/src/python/data/delta.rs +++ b/nautilus_core/model/src/python/data/delta.rs @@ -77,41 +77,6 @@ impl OrderBookDelta { format!("{self:?}") } - #[getter] - fn instrument_id(&self) -> InstrumentId { - self.instrument_id - } - - #[getter] - fn action(&self) -> BookAction { - self.action - } - - #[getter] - fn order(&self) -> BookOrder { - self.order - } - - #[getter] - fn flags(&self) -> u8 { - self.flags - } - - #[getter] - fn sequence(&self) -> u64 { - self.sequence - } - - #[getter] - fn ts_event(&self) -> UnixNanos { - self.ts_event - } - - #[getter] - fn ts_init(&self) -> UnixNanos { - self.ts_init - } - #[staticmethod] #[pyo3(name = "fully_qualified_name")] fn py_fully_qualified_name() -> String { diff --git a/nautilus_core/model/src/python/data/depth.rs b/nautilus_core/model/src/python/data/depth.rs index 6e5aaa2a6d59..37392c042255 100644 --- a/nautilus_core/model/src/python/data/depth.rs +++ b/nautilus_core/model/src/python/data/depth.rs @@ -84,51 +84,6 @@ impl OrderBookDepth10 { format!("{self:?}") } - #[getter] - fn instrument_id(&self) -> InstrumentId { - self.instrument_id - } - - #[getter] - fn bids(&self) -> [BookOrder; DEPTH10_LEN] { - self.bids - } - - #[getter] - fn asks(&self) -> [BookOrder; DEPTH10_LEN] { - self.asks - } - - #[getter] - fn bid_counts(&self) -> [u32; DEPTH10_LEN] { - self.bid_counts - } - - #[getter] - fn ask_counts(&self) -> [u32; DEPTH10_LEN] { - self.ask_counts - } - - #[getter] - fn flags(&self) -> u8 { - self.flags - } - - #[getter] - fn sequence(&self) -> u64 { - self.sequence - } - - #[getter] - fn ts_event(&self) -> UnixNanos { - self.ts_event - } - - #[getter] - fn ts_init(&self) -> UnixNanos { - self.ts_init - } - #[staticmethod] #[pyo3(name = "fully_qualified_name")] fn py_fully_qualified_name() -> String { diff --git a/nautilus_core/model/src/python/data/order.rs b/nautilus_core/model/src/python/data/order.rs index 52f87ea66ba7..3792d8e83763 100644 --- a/nautilus_core/model/src/python/data/order.rs +++ b/nautilus_core/model/src/python/data/order.rs @@ -60,26 +60,6 @@ impl BookOrder { format!("{self:?}") } - #[getter] - fn side(&self) -> OrderSide { - self.side - } - - #[getter] - fn price(&self) -> Price { - self.price - } - - #[getter] - fn size(&self) -> Quantity { - self.size - } - - #[getter] - fn order_id(&self) -> u64 { - self.order_id - } - #[staticmethod] #[pyo3(name = "fully_qualified_name")] fn py_fully_qualified_name() -> String { diff --git a/nautilus_core/model/src/python/data/quote.rs b/nautilus_core/model/src/python/data/quote.rs index 15380d6b5d8a..e43e195c3e9c 100644 --- a/nautilus_core/model/src/python/data/quote.rs +++ b/nautilus_core/model/src/python/data/quote.rs @@ -157,41 +157,6 @@ impl QuoteTick { format!("{}({})", stringify!(QuoteTick), self) } - #[getter] - fn instrument_id(&self) -> InstrumentId { - self.instrument_id - } - - #[getter] - fn bid_price(&self) -> Price { - self.bid_price - } - - #[getter] - fn ask_price(&self) -> Price { - self.ask_price - } - - #[getter] - fn bid_size(&self) -> Quantity { - self.bid_size - } - - #[getter] - fn ask_size(&self) -> Quantity { - self.ask_size - } - - #[getter] - fn ts_event(&self) -> UnixNanos { - self.ts_event - } - - #[getter] - fn ts_init(&self) -> UnixNanos { - self.ts_init - } - #[staticmethod] #[pyo3(name = "fully_qualified_name")] fn py_fully_qualified_name() -> String { diff --git a/nautilus_core/model/src/python/data/trade.rs b/nautilus_core/model/src/python/data/trade.rs index 84c12be62c8b..67c75d712222 100644 --- a/nautilus_core/model/src/python/data/trade.rs +++ b/nautilus_core/model/src/python/data/trade.rs @@ -148,41 +148,6 @@ impl TradeTick { format!("{}({})", stringify!(TradeTick), self) } - #[getter] - fn instrument_id(&self) -> InstrumentId { - self.instrument_id - } - - #[getter] - fn price(&self) -> Price { - self.price - } - - #[getter] - fn size(&self) -> Quantity { - self.size - } - - #[getter] - fn aggressor_side(&self) -> AggressorSide { - self.aggressor_side - } - - #[getter] - fn trade_id(&self) -> TradeId { - self.trade_id - } - - #[getter] - fn ts_event(&self) -> UnixNanos { - self.ts_event - } - - #[getter] - fn ts_init(&self) -> UnixNanos { - self.ts_init - } - #[staticmethod] #[pyo3(name = "fully_qualified_name")] fn py_fully_qualified_name() -> String { From c9705cf3784b756294f3cf41ef61d9854a03ade4 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 28 Jan 2024 23:09:17 +1100 Subject: [PATCH 23/42] Cleanup pyo3 testkit and type stubs --- nautilus_trader/core/nautilus_pyo3.pyi | 3 +- .../test_kit/rust/accounting_pyo3.py | 1 + nautilus_trader/test_kit/rust/events_pyo3.py | 34 +++++++-------- .../test_kit/rust/instruments_pyo3.py | 41 +++++++++---------- nautilus_trader/test_kit/rust/types_pyo3.py | 16 ++++---- 5 files changed, 49 insertions(+), 46 deletions(-) diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 50389f6268b3..0f06f0a8f593 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -788,6 +788,7 @@ class Quantity: def as_decimal(self) -> Decimal: ... def as_double(self) -> float: ... def to_formatted_str(self) -> str: ... + class AccountBalance: def __init__(self, total: Money, locked: Money, free: Money): ... @classmethod @@ -795,7 +796,7 @@ class AccountBalance: def to_dict(self) -> dict[str, str]: ... class MarginBalance: - def __init__(self, initial: Money, maintenance: Money, instrument: InstrumentId): ... + def __init__(self, initial: Money, maintenance: Money, instrument_id: InstrumentId): ... @classmethod def from_dict(cls, values: dict[str, str]) -> MarginBalance: ... def to_dict(self) -> dict[str, str]: ... diff --git a/nautilus_trader/test_kit/rust/accounting_pyo3.py b/nautilus_trader/test_kit/rust/accounting_pyo3.py index 6daee6542bca..7f43f711e5ac 100644 --- a/nautilus_trader/test_kit/rust/accounting_pyo3.py +++ b/nautilus_trader/test_kit/rust/accounting_pyo3.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- + from nautilus_trader.core.nautilus_pyo3 import MarginAccount from nautilus_trader.test_kit.rust.events_pyo3 import TestEventsProviderPyo3 diff --git a/nautilus_trader/test_kit/rust/events_pyo3.py b/nautilus_trader/test_kit/rust/events_pyo3.py index 232a6f84ef0b..4f010ed4136e 100644 --- a/nautilus_trader/test_kit/rust/events_pyo3.py +++ b/nautilus_trader/test_kit/rust/events_pyo3.py @@ -50,7 +50,7 @@ from nautilus_trader.test_kit.rust.types_pyo3 import TestTypesProviderPyo3 -uuid = "91762096-b188-49ea-8562-8d8a4cc22ff2" +_STUB_UUID = "91762096-b188-49ea-8562-8d8a4cc22ff2" class TestEventsProviderPyo3: @@ -96,7 +96,7 @@ def order_denied_max_submit_rate() -> OrderDenied: instrument_id=TestIdProviderPyo3.audusd_id(), client_order_id=TestIdProviderPyo3.client_order_id(), reason="Exceeded MAX_ORDER_SUBMIT_RATE", - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, ) @@ -110,7 +110,7 @@ def order_rejected_insufficient_margin() -> OrderRejected: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), reason="INSUFFICIENT_MARGIN", - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -134,7 +134,7 @@ def order_filled_buy_limit() -> OrderFilled: liquidity_side=LiquiditySide.MAKER, position_id=PositionId("2"), commission=Money.from_str("12.2 USDT"), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -155,7 +155,7 @@ def order_initialized() -> OrderInitialized: reduce_only=True, quote_quantity=False, reconciliation=False, - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), emulation_trigger=TriggerType.BID_ASK, trigger_instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), price=Price.from_str("1520.10"), @@ -178,7 +178,7 @@ def order_triggered() -> OrderTriggered: strategy_id=TestIdProviderPyo3.strategy_id(), instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, venue_order_id=TestIdProviderPyo3.venue_order_id(), @@ -194,7 +194,7 @@ def order_submitted() -> OrderSubmitted: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, ) @@ -206,7 +206,7 @@ def order_emulated() -> OrderEmulated: strategy_id=TestIdProviderPyo3.strategy_id(), instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, ) @@ -219,7 +219,7 @@ def order_released() -> OrderReleased: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), released_price=Price.from_str("22000.0"), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, ) @@ -232,7 +232,7 @@ def order_updated() -> OrderUpdated: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), quantity=Quantity.from_str("1.5"), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -250,7 +250,7 @@ def order_pending_update() -> OrderPendingUpdate: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -265,7 +265,7 @@ def order_pending_cancel() -> OrderPendingCancel: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -281,7 +281,7 @@ def order_modified_rejected(): client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), reason="ORDER_DOES_NOT_EXIST", ts_init=0, ts_event=0, @@ -297,7 +297,7 @@ def order_accepted() -> OrderAccepted: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -313,7 +313,7 @@ def order_cancel_rejected() -> OrderCancelRejected: account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), reason="ORDER_DOES_NOT_EXIST", - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -328,7 +328,7 @@ def order_canceled() -> OrderCanceled: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, @@ -343,7 +343,7 @@ def order_expired() -> OrderExpired: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(uuid), + event_id=UUID4(_STUB_UUID), ts_init=0, ts_event=0, reconciliation=False, diff --git a/nautilus_trader/test_kit/rust/instruments_pyo3.py b/nautilus_trader/test_kit/rust/instruments_pyo3.py index 47b8d6dfbdd6..44ba856cdd7a 100644 --- a/nautilus_trader/test_kit/rust/instruments_pyo3.py +++ b/nautilus_trader/test_kit/rust/instruments_pyo3.py @@ -13,7 +13,6 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- - from decimal import Decimal import pandas as pd @@ -37,10 +36,10 @@ from nautilus_trader.test_kit.rust.types_pyo3 import TestTypesProviderPyo3 -USD = TestTypesProviderPyo3.currency_usd() -USDT = TestTypesProviderPyo3.currency_usdt() -BTC = TestTypesProviderPyo3.currency_btc() -ETH = TestTypesProviderPyo3.currency_eth() +_USD = TestTypesProviderPyo3.currency_usd() +_USDT = TestTypesProviderPyo3.currency_usdt() +_BTC = TestTypesProviderPyo3.currency_btc() +_ETH = TestTypesProviderPyo3.currency_eth() class TestInstrumentProviderPyo3: @@ -49,9 +48,9 @@ def ethusdt_perp_binance() -> CryptoPerpetual: return CryptoPerpetual( id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), symbol=Symbol("ETHUSDT"), - base_currency=ETH, - quote_currency=USDT, - settlement_currency=USDT, + base_currency=_ETH, + quote_currency=_USDT, + settlement_currency=_USDT, is_inverse=False, price_precision=2, size_precision=0, @@ -61,7 +60,7 @@ def ethusdt_perp_binance() -> CryptoPerpetual: max_quantity=Quantity.from_str("10000"), min_quantity=Quantity.from_str("0.001"), max_notional=None, - min_notional=Money(10.0, USDT), + min_notional=Money(10.0, _USDT), max_price=Price.from_str("15000.0"), min_price=Price.from_str("1.0"), margin_init=Decimal("1.00"), @@ -80,9 +79,9 @@ def xbtusd_bitmex() -> CryptoPerpetual: venue=Venue("BITMEX"), ), symbol=Symbol("XBTUSD"), - base_currency=BTC, - quote_currency=USD, - settlement_currency=BTC, + base_currency=_BTC, + quote_currency=_USD, + settlement_currency=_BTC, is_inverse=True, price_precision=1, size_precision=0, @@ -90,8 +89,8 @@ def xbtusd_bitmex() -> CryptoPerpetual: size_increment=Quantity.from_int(1), max_quantity=None, min_quantity=None, - max_notional=Money(10_000_000.00, USD), - min_notional=Money(1.00, USD), + max_notional=Money(10_000_000.00, _USD), + min_notional=Money(1.00, _USD), max_price=Price.from_str("1000000.0"), min_price=Price(0.5, precision=1), margin_init=Decimal("0.01"), @@ -116,9 +115,9 @@ def btcusdt_future_binance( return CryptoFuture( id=InstrumentId.from_str(instrument_id_str), raw_symbol=Symbol("BTCUSDT"), - underlying=BTC, - quote_currency=USDT, - settlement_currency=USDT, + underlying=_BTC, + quote_currency=_USDT, + settlement_currency=_USDT, activation_ns=activation.value, expiration_ns=expiration.value, price_precision=2, @@ -142,7 +141,7 @@ def btcusdt_binance() -> CurrencyPair: id=InstrumentId.from_str("BTCUSDT.BINANCE"), raw_symbol=Symbol("BTCUSDT"), base_currency=TestTypesProviderPyo3.currency_btc(), - quote_currency=USDT, + quote_currency=_USDT, price_precision=2, size_precision=6, price_increment=Price.from_str("0.01"), @@ -178,7 +177,7 @@ def aapl_option( activation_ns=activation.value, expiration_ns=expiration.value, strike_price=Price.from_str("149.0"), - currency=USDT, + currency=_USDT, price_precision=2, price_increment=Price.from_str("0.01"), multiplier=Quantity.from_int(1), @@ -197,7 +196,7 @@ def aapl_equity() -> Equity: id=InstrumentId.from_str("AAPL.XNAS"), raw_symbol=Symbol("AAPL"), isin="US0378331005", - currency=USD, + currency=_USD, price_precision=2, price_increment=Price.from_str("0.01"), lot_size=Quantity.from_int(100), @@ -225,7 +224,7 @@ def futures_contract_es( underlying="ES", activation_ns=activation.value, expiration_ns=expiration.value, - currency=USD, + currency=_USD, price_precision=2, price_increment=Price.from_str("0.01"), multiplier=Quantity.from_int(1), diff --git a/nautilus_trader/test_kit/rust/types_pyo3.py b/nautilus_trader/test_kit/rust/types_pyo3.py index d066ac6f5260..fed86abd52f3 100644 --- a/nautilus_trader/test_kit/rust/types_pyo3.py +++ b/nautilus_trader/test_kit/rust/types_pyo3.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- + from nautilus_trader.core.nautilus_pyo3 import AccountBalance from nautilus_trader.core.nautilus_pyo3 import Currency +from nautilus_trader.core.nautilus_pyo3 import InstrumentId from nautilus_trader.core.nautilus_pyo3 import MarginBalance from nautilus_trader.core.nautilus_pyo3 import Money from nautilus_trader.test_kit.rust.identifiers_pyo3 import TestIdProviderPyo3 @@ -46,16 +48,16 @@ def currency_eth() -> Currency: @staticmethod def account_balance( - total=Money.from_str("1525000 USD"), - locked=Money.from_str("25000 USD"), - free=Money.from_str("1500000 USD"), + total: Money = Money.from_str("1525000 USD"), + locked: Money = Money.from_str("25000 USD"), + free: Money = Money.from_str("1500000 USD"), ) -> AccountBalance: return AccountBalance(total, locked, free) @staticmethod def margin_balance( - initial=Money(1, Currency.from_str("USD")), - maintenance=Money(1, Currency.from_str("USD")), - instrument=TestIdProviderPyo3.audusd_id(), + initial: Money = Money(1, Currency.from_str("USD")), + maintenance: Money = Money(1, Currency.from_str("USD")), + instrument_id: InstrumentId = TestIdProviderPyo3.audusd_id(), ) -> MarginBalance: - return MarginBalance(initial, maintenance, instrument) + return MarginBalance(initial, maintenance, instrument_id) From a7bc5fa4676d6cbd3744ac75b9364cda4bc3cdf9 Mon Sep 17 00:00:00 2001 From: Ben Singleton Date: Sun, 28 Jan 2024 11:37:48 -0500 Subject: [PATCH 24/42] Fix position EWrapper override --- .../interactive_brokers/client/account.py | 40 +++++++------------ 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/nautilus_trader/adapters/interactive_brokers/client/account.py b/nautilus_trader/adapters/interactive_brokers/client/account.py index 739f5810a6c4..f5f1bd887feb 100644 --- a/nautilus_trader/adapters/interactive_brokers/client/account.py +++ b/nautilus_trader/adapters/interactive_brokers/client/account.py @@ -98,32 +98,6 @@ def unsubscribe_account_summary(self, account_id: str) -> None: else: self._log.debug(f"Subscription doesn't exist for {name}") - def process_position( - self, - account_id: str, - contract: IBContract, - position: Decimal, - avg_cost: float, - ) -> None: - """ - Process position data for an account. - - Parameters - ---------- - account_id : str - The account identifier - contract : IBContract - The contract details for the position. - position : Decimal - The quantity of the position. - avg_cost : float - The average cost of the position. - - """ - self.logAnswer(current_fn_name(), vars()) - if request := self._requests.get(name="OpenPositions"): - request.result.append(IBPosition(account_id, contract, position, avg_cost)) - async def get_positions(self, account_id: str) -> list[Position] | None: """ Fetch open positions for a specified account. @@ -191,6 +165,20 @@ def managedAccounts(self, accounts_list: str) -> None: self._log.info("`is_ib_ready` set by managedAccounts", LogColor.BLUE) self._is_ib_ready.set() + def position( + self, + account_id: str, + contract: IBContract, + position: Decimal, + avg_cost: float, + ) -> None: + """ + Provide the portfolio's open positions. + """ + self.logAnswer(current_fn_name(), vars()) + if request := self._requests.get(name="OpenPositions"): + request.result.append(IBPosition(account_id, contract, position, avg_cost)) + def positionEnd(self) -> None: """ Indicate that all the positions have been transmitted. From ede228d9b5b581fc9dea8136b1e07bc3ee109f84 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 29 Jan 2024 07:13:04 +1100 Subject: [PATCH 25/42] Update release notes --- RELEASES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 91bda009b91c..66ae4d87d0d7 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -9,7 +9,7 @@ None None ### Fixes -None +- Fixed Interactive Brokers get account positions bug (#1475), thanks @benjaminsingleton --- From f58e27bd0dfd2b23210fb32a782bf3336980867d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 29 Jan 2024 07:20:37 +1100 Subject: [PATCH 26/42] Update dependencies including indexmap --- nautilus_core/Cargo.lock | 28 ++++++++--------- nautilus_core/Cargo.toml | 2 +- nautilus_core/common/src/msgbus.rs | 6 ++-- poetry.lock | 48 +++++++++++++++--------------- pyproject.toml | 2 +- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 8bd2e496d005..8a5a157d936e 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -266,7 +266,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.1.0", + "indexmap 2.2.0", "lexical-core", "num", "serde", @@ -1150,7 +1150,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.3", - "indexmap 2.1.0", + "indexmap 2.2.0", "itertools 0.12.0", "log", "num_cpus", @@ -1266,7 +1266,7 @@ dependencies = [ "half", "hashbrown 0.14.3", "hex", - "indexmap 2.1.0", + "indexmap 2.2.0", "itertools 0.12.0", "log", "md-5", @@ -1299,7 +1299,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.3", - "indexmap 2.1.0", + "indexmap 2.2.0", "itertools 0.12.0", "log", "once_cell", @@ -1765,7 +1765,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.1.0", + "indexmap 2.2.0", "slab", "tokio", "tokio-util", @@ -1784,7 +1784,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.1.0", + "indexmap 2.2.0", "slab", "tokio", "tokio-util", @@ -2092,9 +2092,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "cf2a4f498956c7723dc280afc6a37d0dec50b39a29e232c6187ce4503703e8c2" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2422,7 +2422,7 @@ dependencies = [ "criterion", "databento", "dbn", - "indexmap 2.1.0", + "indexmap 2.2.0", "itoa", "log", "nautilus-common", @@ -2464,7 +2464,7 @@ dependencies = [ "anyhow", "cbindgen", "chrono", - "indexmap 2.1.0", + "indexmap 2.2.0", "log", "nautilus-core", "nautilus-model", @@ -2539,7 +2539,7 @@ dependencies = [ "evalexpr", "float-cmp", "iai", - "indexmap 2.1.0", + "indexmap 2.2.0", "nautilus-core", "once_cell", "pyo3", @@ -3013,7 +3013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.1.0", + "indexmap 2.2.0", ] [[package]] @@ -4182,7 +4182,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.1.0", + "indexmap 2.2.0", "log", "memchr", "once_cell", @@ -4778,7 +4778,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.0", "toml_datetime", "winnow", ] diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 8d965f11231e..2d09c5c30ef6 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -26,7 +26,7 @@ documentation = "https://docs.nautilustrader.io" anyhow = "1.0.79" chrono = "0.4.33" futures = "0.3.30" -indexmap = "2.1.0" +indexmap = "2.2.0" itoa = "1.0.10" once_cell = "1.19.0" log = { version = "0.4.20", features = ["std", "kv_unstable", "serde", "release_max_level_debug"] } diff --git a/nautilus_core/common/src/msgbus.rs b/nautilus_core/common/src/msgbus.rs index 85d0136e2bd8..410513a8b351 100644 --- a/nautilus_core/common/src/msgbus.rs +++ b/nautilus_core/common/src/msgbus.rs @@ -276,7 +276,7 @@ impl MessageBus { /// Deregisters the given `handler` for the `endpoint` address. pub fn deregister(&mut self, endpoint: &str) { // Removes entry if it exists for endpoint - self.endpoints.remove(&Ustr::from(endpoint)); + self.endpoints.shift_remove(&Ustr::from(endpoint)); } /// Subscribes the given `handler` to the `topic`. @@ -309,7 +309,7 @@ impl MessageBus { /// Unsubscribes the given `handler` from the `topic`. pub fn unsubscribe(&mut self, topic: &str, handler: MessageHandler) { let sub = Subscription::new(Ustr::from(topic), handler, self.subscriptions.len(), None); - self.subscriptions.remove(&sub); + self.subscriptions.shift_remove(&sub); } /// Returns the handler for the given `endpoint`. @@ -345,7 +345,7 @@ impl MessageBus { /// index. #[must_use] pub fn response_handler(&mut self, correlation_id: &UUID4) -> Option { - self.correlation_index.remove(correlation_id) + self.correlation_index.shift_remove(correlation_id) } #[must_use] diff --git a/poetry.lock b/poetry.lock index e27715d383dc..27b2fcdaad7a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -202,33 +202,33 @@ msgspec = ">=0.18.5" [[package]] name = "black" -version = "24.1.0" +version = "24.1.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:94d5280d020dadfafc75d7cae899609ed38653d3f5e82e7ce58f75e76387ed3d"}, - {file = "black-24.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aaf9aa85aaaa466bf969e7dd259547f4481b712fe7ee14befeecc152c403ee05"}, - {file = "black-24.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec489cae76eac3f7573629955573c3a0e913641cafb9e3bfc87d8ce155ebdb29"}, - {file = "black-24.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:a5a0100b4bdb3744dd68412c3789f472d822dc058bb3857743342f8d7f93a5a7"}, - {file = "black-24.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6cc5a6ba3e671cfea95a40030b16a98ee7dc2e22b6427a6f3389567ecf1b5262"}, - {file = "black-24.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0e367759062dcabcd9a426d12450c6d61faf1704a352a49055a04c9f9ce8f5a"}, - {file = "black-24.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be305563ff4a2dea813f699daaffac60b977935f3264f66922b1936a5e492ee4"}, - {file = "black-24.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:6a8977774929b5db90442729f131221e58cc5d8208023c6af9110f26f75b6b20"}, - {file = "black-24.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d74d4d0da276fbe3b95aa1f404182562c28a04402e4ece60cf373d0b902f33a0"}, - {file = "black-24.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39addf23f7070dbc0b5518cdb2018468ac249d7412a669b50ccca18427dba1f3"}, - {file = "black-24.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:827a7c0da520dd2f8e6d7d3595f4591aa62ccccce95b16c0e94bb4066374c4c2"}, - {file = "black-24.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0cd59d01bf3306ff7e3076dd7f4435fcd2fafe5506a6111cae1138fc7de52382"}, - {file = "black-24.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf8dd261ee82df1abfb591f97e174345ab7375a55019cc93ad38993b9ff5c6ad"}, - {file = "black-24.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:82d9452aeabd51d1c8f0d52d4d18e82b9f010ecb30fd55867b5ff95904f427ff"}, - {file = "black-24.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aede09f72b2a466e673ee9fca96e4bccc36f463cac28a35ce741f0fd13aea8b"}, - {file = "black-24.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:780f13d03066a7daf1707ec723fdb36bd698ffa29d95a2e7ef33a8dd8fe43b5c"}, - {file = "black-24.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a15670c650668399c4b5eae32e222728185961d6ef6b568f62c1681d57b381ba"}, - {file = "black-24.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1e0fa70b8464055069864a4733901b31cbdbe1273f63a24d2fa9d726723d45ac"}, - {file = "black-24.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fa8d9aaa22d846f8c0f7f07391148e5e346562e9b215794f9101a8339d8b6d8"}, - {file = "black-24.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:f0dfbfbacfbf9cd1fac7a5ddd3e72510ffa93e841a69fcf4a6358feab1685382"}, - {file = "black-24.1.0-py3-none-any.whl", hash = "sha256:5134a6f6b683aa0a5592e3fd61dd3519d8acd953d93e2b8b76f9981245b65594"}, - {file = "black-24.1.0.tar.gz", hash = "sha256:30fbf768cd4f4576598b1db0202413fafea9a227ef808d1a12230c643cefe9fc"}, + {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, + {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, + {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, + {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, + {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, + {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, + {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, + {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, + {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, + {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, + {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, + {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, + {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, + {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, + {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, + {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, + {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, + {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, + {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, + {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, + {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, + {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, ] [package.dependencies] @@ -2594,4 +2594,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "cd49b4783329bf0b0e1f170c3d92a37527b103b5e32f255e82ab4063f9261c64" +content-hash = "ccf3e64dcf9a2e22ee29c2a572fef6869e38bfcd6b096874d080adcd6f49612f" diff --git a/pyproject.toml b/pyproject.toml index e1073558272e..0dc009dda559 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ ib = ["nautilus_ibapi", "async-timeout", "defusedxml"] optional = true [tool.poetry.group.dev.dependencies] -black = "^24.1.0" +black = "^24.1.1" docformatter = "^1.7.5" mypy = "^1.8.0" pandas-stubs = "^2.1.4" From f242f9c503e5f59eb3d277fb1496e5a8e267565b Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 29 Jan 2024 07:33:27 +1100 Subject: [PATCH 27/42] Add bypass_logging session fixture --- tests/conftest.py | 13 +++++++++++++ tests/unit_tests/common/test_logging.py | 8 -------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b18475072491..bf363c0b1356 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ import pytest +from nautilus_trader.common.component import init_logging from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.instruments import CurrencyPair @@ -23,6 +24,18 @@ from nautilus_trader.test_kit.providers import TestInstrumentProvider +@pytest.fixture(scope="session", autouse=True) +def bypass_logging() -> None: + """ + Fixture to bypass logging for all tests. + + `autouse=True` will mean this function is run prior to every test. To disable this + to debug specific tests, simply comment this out. + + """ + init_logging(bypass=True) + + @pytest.fixture(name="audusd_instrument") def fixture_audusd_instrument() -> CurrencyPair: return TestInstrumentProvider.default_fx_ccy("AUD/USD", Venue("SIM")) diff --git a/tests/unit_tests/common/test_logging.py b/tests/unit_tests/common/test_logging.py index c675bf71aac3..d0e4702a1add 100644 --- a/tests/unit_tests/common/test_logging.py +++ b/tests/unit_tests/common/test_logging.py @@ -16,7 +16,6 @@ import pytest from nautilus_trader.common.component import Logger -from nautilus_trader.common.component import init_logging from nautilus_trader.common.enums import LogColor from nautilus_trader.common.enums import LogLevel from nautilus_trader.common.enums import log_level_from_str @@ -69,7 +68,6 @@ def test_name(self): def test_log_debug_messages_to_console(self): # Arrange - init_logging(level_stdout=LogLevel.DEBUG, bypass=True) logger = Logger(name="TEST_LOGGER") # Act @@ -80,7 +78,6 @@ def test_log_debug_messages_to_console(self): def test_log_info_messages_to_console(self): # Arrange - init_logging(level_stdout=LogLevel.INFO, bypass=True) logger = Logger(name="TEST_LOGGER") # Act @@ -91,7 +88,6 @@ def test_log_info_messages_to_console(self): def test_log_info_messages_to_console_with_blue_colour(self): # Arrange - init_logging(level_stdout=LogLevel.INFO, bypass=True) logger = Logger(name="TEST_LOGGER") # Act @@ -102,7 +98,6 @@ def test_log_info_messages_to_console_with_blue_colour(self): def test_log_info_messages_to_console_with_green_colour(self): # Arrange - init_logging(level_stdout=LogLevel.INFO, bypass=True) logger = Logger(name="TEST_LOGGER") # Act @@ -113,7 +108,6 @@ def test_log_info_messages_to_console_with_green_colour(self): def test_log_warning_messages_to_console(self): # Arrange - init_logging(level_stdout=LogLevel.WARNING, bypass=True) logger = Logger(name="TEST_LOGGER") # Act @@ -124,7 +118,6 @@ def test_log_warning_messages_to_console(self): def test_log_error_messages_to_console(self): # Arrange - init_logging(level_stdout=LogLevel.ERROR, bypass=True) logger = Logger(name="TEST_LOGGER") # Act @@ -135,7 +128,6 @@ def test_log_error_messages_to_console(self): def test_log_exception_messages_to_console(self): # Arrange - init_logging(level_stdout=LogLevel.ERROR) logger = Logger(name="TEST_LOGGER") # Act From b23af845e1371baba68dc251ed09879e12682a4b Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 29 Jan 2024 07:35:49 +1100 Subject: [PATCH 28/42] Cleanup stub UUID4 --- nautilus_trader/test_kit/rust/events_pyo3.py | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/nautilus_trader/test_kit/rust/events_pyo3.py b/nautilus_trader/test_kit/rust/events_pyo3.py index 4f010ed4136e..94777329e28a 100644 --- a/nautilus_trader/test_kit/rust/events_pyo3.py +++ b/nautilus_trader/test_kit/rust/events_pyo3.py @@ -50,7 +50,7 @@ from nautilus_trader.test_kit.rust.types_pyo3 import TestTypesProviderPyo3 -_STUB_UUID = "91762096-b188-49ea-8562-8d8a4cc22ff2" +_STUB_UUID4 = UUID4("91762096-b188-49ea-8562-8d8a4cc22ff2") class TestEventsProviderPyo3: @@ -96,7 +96,7 @@ def order_denied_max_submit_rate() -> OrderDenied: instrument_id=TestIdProviderPyo3.audusd_id(), client_order_id=TestIdProviderPyo3.client_order_id(), reason="Exceeded MAX_ORDER_SUBMIT_RATE", - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, ) @@ -110,7 +110,7 @@ def order_rejected_insufficient_margin() -> OrderRejected: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), reason="INSUFFICIENT_MARGIN", - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -134,7 +134,7 @@ def order_filled_buy_limit() -> OrderFilled: liquidity_side=LiquiditySide.MAKER, position_id=PositionId("2"), commission=Money.from_str("12.2 USDT"), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -155,7 +155,7 @@ def order_initialized() -> OrderInitialized: reduce_only=True, quote_quantity=False, reconciliation=False, - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, emulation_trigger=TriggerType.BID_ASK, trigger_instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), price=Price.from_str("1520.10"), @@ -178,7 +178,7 @@ def order_triggered() -> OrderTriggered: strategy_id=TestIdProviderPyo3.strategy_id(), instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, venue_order_id=TestIdProviderPyo3.venue_order_id(), @@ -194,7 +194,7 @@ def order_submitted() -> OrderSubmitted: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, ) @@ -206,7 +206,7 @@ def order_emulated() -> OrderEmulated: strategy_id=TestIdProviderPyo3.strategy_id(), instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, ) @@ -219,7 +219,7 @@ def order_released() -> OrderReleased: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), released_price=Price.from_str("22000.0"), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, ) @@ -232,7 +232,7 @@ def order_updated() -> OrderUpdated: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), quantity=Quantity.from_str("1.5"), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -250,7 +250,7 @@ def order_pending_update() -> OrderPendingUpdate: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -265,7 +265,7 @@ def order_pending_cancel() -> OrderPendingCancel: instrument_id=TestIdProviderPyo3.ethusdt_binance_id(), client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -281,7 +281,7 @@ def order_modified_rejected(): client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, reason="ORDER_DOES_NOT_EXIST", ts_init=0, ts_event=0, @@ -297,7 +297,7 @@ def order_accepted() -> OrderAccepted: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -313,7 +313,7 @@ def order_cancel_rejected() -> OrderCancelRejected: account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), reason="ORDER_DOES_NOT_EXIST", - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -328,7 +328,7 @@ def order_canceled() -> OrderCanceled: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, @@ -343,7 +343,7 @@ def order_expired() -> OrderExpired: client_order_id=TestIdProviderPyo3.client_order_id(), account_id=TestIdProviderPyo3.account_id(), venue_order_id=TestIdProviderPyo3.venue_order_id(), - event_id=UUID4(_STUB_UUID), + event_id=_STUB_UUID4, ts_init=0, ts_event=0, reconciliation=False, From c0410d6b184c780f6e619bd367c7321db2b3c4f2 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 29 Jan 2024 07:58:19 +1100 Subject: [PATCH 29/42] Add pyo3 conversion performance benchmarks --- .../test_perf_pyo3_conversion.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/performance_tests/test_perf_pyo3_conversion.py diff --git a/tests/performance_tests/test_perf_pyo3_conversion.py b/tests/performance_tests/test_perf_pyo3_conversion.py new file mode 100644 index 000000000000..1c3ee0fac5d8 --- /dev/null +++ b/tests/performance_tests/test_perf_pyo3_conversion.py @@ -0,0 +1,108 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +from nautilus_trader.model.data import Bar +from nautilus_trader.model.data import OrderBookDelta +from nautilus_trader.model.data import QuoteTick +from nautilus_trader.model.data import TradeTick +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 + + +def test_pyo3_delta_to_legacy_cython(benchmark): + pyo3_delta = TestDataProviderPyo3.order_book_delta() + + benchmark.pedantic( + target=OrderBookDelta.from_pyo3, + args=(pyo3_delta,), + rounds=100_000, + iterations=1, + ) + + +def test_pyo3_deltas_to_legacy_cython_list(benchmark): + pyo3_deltas = [TestDataProviderPyo3.order_book_delta()] * 10_000 + + benchmark.pedantic( + target=OrderBookDelta.from_pyo3_list, + args=(pyo3_deltas,), + rounds=10, + iterations=1, + ) + + +def test_pyo3_quote_to_legacy_cython(benchmark): + pyo3_quote = TestDataProviderPyo3.quote_tick() + + benchmark.pedantic( + target=QuoteTick.from_pyo3, + args=(pyo3_quote,), + rounds=100_000, + iterations=1, + ) + + +def test_pyo3_quotes_to_legacy_cython_list(benchmark): + pyo3_quotes = [TestDataProviderPyo3.quote_tick()] * 10_000 + + benchmark.pedantic( + target=QuoteTick.from_pyo3_list, + args=(pyo3_quotes,), + rounds=10, + iterations=1, + ) + + +def test_pyo3_trade_to_legacy_cython(benchmark): + pyo3_trade = TestDataProviderPyo3.trade_tick() + + benchmark.pedantic( + target=TradeTick.from_pyo3, + args=(pyo3_trade,), + rounds=100_000, + iterations=1, + ) + + +def test_pyo3_trades_to_legacy_cython_list(benchmark): + pyo3_trades = [TestDataProviderPyo3.trade_tick()] * 10_000 + + benchmark.pedantic( + target=TradeTick.from_pyo3_list, + args=(pyo3_trades,), + rounds=10, + iterations=1, + ) + + +def test_pyo3_bar_to_legacy_cython(benchmark): + pyo3_bar = TestDataProviderPyo3.bar_5decimal() + + benchmark.pedantic( + target=Bar.from_pyo3, + args=(pyo3_bar,), + rounds=100_000, + iterations=1, + ) + + +def test_pyo3_bars_to_legacy_cython_list(benchmark): + pyo3_bars = [TestDataProviderPyo3.bar_5decimal()] * 10_000 + + benchmark.pedantic( + target=Bar.from_pyo3_list, + args=(pyo3_bars,), + rounds=10, + iterations=1, + ) From 57ee1fe334065bf2dc88c543d90a9f2161493fce Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 30 Jan 2024 16:39:13 +1100 Subject: [PATCH 30/42] Fix TimeBarAggregator interval types on build --- RELEASES.md | 1 + nautilus_trader/data/aggregation.pyx | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 66ae4d87d0d7..6d5dba0215a2 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,7 @@ None ### Fixes - Fixed Interactive Brokers get account positions bug (#1475), thanks @benjaminsingleton +- Fixed `TimeBarAggregator` handling of interval types on build --- diff --git a/nautilus_trader/data/aggregation.pyx b/nautilus_trader/data/aggregation.pyx index 9bd2475b3d6b..c5b482cad520 100644 --- a/nautilus_trader/data/aggregation.pyx +++ b/nautilus_trader/data/aggregation.pyx @@ -717,9 +717,9 @@ cdef class TimeBarAggregator(BarAggregator): ts_init = ts_event # Adjusting the timestamp logic based on interval_type - if self._interval_type == "left-open": + if self._is_left_open: ts_event = self._stored_close_ns if self._timestamp_on_close else self._stored_open_ns - elif self._interval_type == "right-open": + else: ts_event = self._stored_open_ns self._build_and_send(ts_event=ts_event, ts_init=ts_init) From 991fb4948a2487a13f9897f989962663f47852f1 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 30 Jan 2024 16:50:47 +1100 Subject: [PATCH 31/42] Update dependencies --- nautilus_core/Cargo.lock | 52 +++++----- nautilus_core/Cargo.toml | 2 +- poetry.lock | 208 +++++++++++++++++++-------------------- pyproject.toml | 2 +- 4 files changed, 132 insertions(+), 132 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 8a5a157d936e..debcac8d0b2c 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -95,9 +95,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anyhow" @@ -266,7 +266,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.2.0", + "indexmap 2.2.1", "lexical-core", "num", "serde", @@ -1150,8 +1150,8 @@ dependencies = [ "glob", "half", "hashbrown 0.14.3", - "indexmap 2.2.0", - "itertools 0.12.0", + "indexmap 2.2.1", + "itertools 0.12.1", "log", "num_cpus", "object_store", @@ -1240,7 +1240,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.3", - "itertools 0.12.0", + "itertools 0.12.1", "log", "regex-syntax 0.8.2", ] @@ -1266,8 +1266,8 @@ dependencies = [ "half", "hashbrown 0.14.3", "hex", - "indexmap 2.2.0", - "itertools 0.12.0", + "indexmap 2.2.1", + "itertools 0.12.1", "log", "md-5", "paste", @@ -1299,8 +1299,8 @@ dependencies = [ "futures", "half", "hashbrown 0.14.3", - "indexmap 2.2.0", - "itertools 0.12.0", + "indexmap 2.2.1", + "itertools 0.12.1", "log", "once_cell", "parking_lot", @@ -1765,7 +1765,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.0", + "indexmap 2.2.1", "slab", "tokio", "tokio-util", @@ -1784,7 +1784,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.0", + "indexmap 2.2.1", "slab", "tokio", "tokio-util", @@ -2092,9 +2092,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2a4f498956c7723dc280afc6a37d0dec50b39a29e232c6187ce4503703e8c2" +checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2140,9 +2140,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -2422,7 +2422,7 @@ dependencies = [ "criterion", "databento", "dbn", - "indexmap 2.2.0", + "indexmap 2.2.1", "itoa", "log", "nautilus-common", @@ -2464,7 +2464,7 @@ dependencies = [ "anyhow", "cbindgen", "chrono", - "indexmap 2.2.0", + "indexmap 2.2.1", "log", "nautilus-core", "nautilus-model", @@ -2539,7 +2539,7 @@ dependencies = [ "evalexpr", "float-cmp", "iai", - "indexmap 2.2.0", + "indexmap 2.2.1", "nautilus-core", "once_cell", "pyo3", @@ -2810,7 +2810,7 @@ dependencies = [ "chrono", "futures", "humantime", - "itertools 0.12.0", + "itertools 0.12.1", "parking_lot", "percent-encoding", "snafu", @@ -3013,7 +3013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.0", + "indexmap 2.2.1", ] [[package]] @@ -3928,9 +3928,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.112" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d1bd37ce2324cf3bf85e5a25f96eb4baf0d5aa6eba43e7ae8958870c4ec48ed" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -4121,7 +4121,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.12.0", + "itertools 0.12.1", "nom", "unicode_categories", ] @@ -4182,7 +4182,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.2.0", + "indexmap 2.2.1", "log", "memchr", "once_cell", @@ -4778,7 +4778,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.2.0", + "indexmap 2.2.1", "toml_datetime", "winnow", ] diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 2d09c5c30ef6..2dc8cb91514f 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -26,7 +26,7 @@ documentation = "https://docs.nautilustrader.io" anyhow = "1.0.79" chrono = "0.4.33" futures = "0.3.30" -indexmap = "2.2.0" +indexmap = "2.2.1" itoa = "1.0.10" once_cell = "1.19.0" log = { version = "0.4.20", features = ["std", "kv_unstable", "serde", "release_max_level_debug"] } diff --git a/poetry.lock b/poetry.lock index 27b2fcdaad7a..b1eadc735d82 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,87 +2,87 @@ [[package]] name = "aiohttp" -version = "3.9.1" +version = "3.9.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590"}, - {file = "aiohttp-3.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0"}, - {file = "aiohttp-3.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83"}, - {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5"}, - {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4"}, - {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f"}, - {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f"}, - {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6"}, - {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f"}, - {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26"}, - {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4"}, - {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d"}, - {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501"}, - {file = "aiohttp-3.9.1-cp310-cp310-win32.whl", hash = "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489"}, - {file = "aiohttp-3.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23"}, - {file = "aiohttp-3.9.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d"}, - {file = "aiohttp-3.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e"}, - {file = "aiohttp-3.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0"}, - {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35"}, - {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff"}, - {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87"}, - {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d"}, - {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01"}, - {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3"}, - {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449"}, - {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2"}, - {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd"}, - {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a"}, - {file = "aiohttp-3.9.1-cp311-cp311-win32.whl", hash = "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544"}, - {file = "aiohttp-3.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587"}, - {file = "aiohttp-3.9.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065"}, - {file = "aiohttp-3.9.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821"}, - {file = "aiohttp-3.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af"}, - {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57"}, - {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5"}, - {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb"}, - {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c"}, - {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66"}, - {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe"}, - {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183"}, - {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b"}, - {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f"}, - {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f"}, - {file = "aiohttp-3.9.1-cp312-cp312-win32.whl", hash = "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed"}, - {file = "aiohttp-3.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213"}, - {file = "aiohttp-3.9.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70"}, - {file = "aiohttp-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672"}, - {file = "aiohttp-3.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1"}, - {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd"}, - {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690"}, - {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca"}, - {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361"}, - {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28"}, - {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014"}, - {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431"}, - {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd"}, - {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a"}, - {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8"}, - {file = "aiohttp-3.9.1-cp38-cp38-win32.whl", hash = "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4"}, - {file = "aiohttp-3.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7"}, - {file = "aiohttp-3.9.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766"}, - {file = "aiohttp-3.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0"}, - {file = "aiohttp-3.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558"}, - {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636"}, - {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499"}, - {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4"}, - {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f"}, - {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a"}, - {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e"}, - {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5"}, - {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0"}, - {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f"}, - {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c"}, - {file = "aiohttp-3.9.1-cp39-cp39-win32.whl", hash = "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7"}, - {file = "aiohttp-3.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf"}, - {file = "aiohttp-3.9.1.tar.gz", hash = "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, ] [package.dependencies] @@ -464,7 +464,7 @@ name = "css-html-js-minify" version = "2.5.5" description = "CSS HTML JS Minifier" optional = false -python-versions = ">=3.6" +python-versions = "*" files = [ {file = "css-html-js-minify-2.5.5.zip", hash = "sha256:4a9f11f7e0496f5284d12111f3ba4ff5ff2023d12f15d195c9c48bd97013746c"}, {file = "css_html_js_minify-2.5.5-py2.py3-none-any.whl", hash = "sha256:3da9d35ac0db8ca648c1b543e0e801d7ca0bab9e6bfd8418fee59d5ae001727a"}, @@ -1809,13 +1809,13 @@ unidecode = ["Unidecode (>=1.1.1)"] [[package]] name = "pytz" -version = "2023.3.post1" +version = "2023.4" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, + {file = "pytz-2023.4-py2.py3-none-any.whl", hash = "sha256:f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a"}, + {file = "pytz-2023.4.tar.gz", hash = "sha256:31d4583c4ed539cd037956140d695e42c033a19e984bfce9964a3f7d59bc2b40"}, ] [[package]] @@ -1913,28 +1913,28 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruff" -version = "0.1.14" +version = "0.1.15" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.1.14-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:96f76536df9b26622755c12ed8680f159817be2f725c17ed9305b472a757cdbb"}, - {file = "ruff-0.1.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ab3f71f64498c7241123bb5a768544cf42821d2a537f894b22457a543d3ca7a9"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7060156ecc572b8f984fd20fd8b0fcb692dd5d837b7606e968334ab7ff0090ab"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a53d8e35313d7b67eb3db15a66c08434809107659226a90dcd7acb2afa55faea"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bea9be712b8f5b4ebed40e1949379cfb2a7d907f42921cf9ab3aae07e6fba9eb"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2270504d629a0b064247983cbc495bed277f372fb9eaba41e5cf51f7ba705a6a"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80258bb3b8909b1700610dfabef7876423eed1bc930fe177c71c414921898efa"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:653230dd00aaf449eb5ff25d10a6e03bc3006813e2cb99799e568f55482e5cae"}, - {file = "ruff-0.1.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b3acc6c4e6928459ba9eb7459dd4f0c4bf266a053c863d72a44c33246bfdbf"}, - {file = "ruff-0.1.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6b3dadc9522d0eccc060699a9816e8127b27addbb4697fc0c08611e4e6aeb8b5"}, - {file = "ruff-0.1.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1c8eca1a47b4150dc0fbec7fe68fc91c695aed798532a18dbb1424e61e9b721f"}, - {file = "ruff-0.1.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:62ce2ae46303ee896fc6811f63d6dabf8d9c389da0f3e3f2bce8bc7f15ef5488"}, - {file = "ruff-0.1.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b2027dde79d217b211d725fc833e8965dc90a16d0d3213f1298f97465956661b"}, - {file = "ruff-0.1.14-py3-none-win32.whl", hash = "sha256:722bafc299145575a63bbd6b5069cb643eaa62546a5b6398f82b3e4403329cab"}, - {file = "ruff-0.1.14-py3-none-win_amd64.whl", hash = "sha256:e3d241aa61f92b0805a7082bd89a9990826448e4d0398f0e2bc8f05c75c63d99"}, - {file = "ruff-0.1.14-py3-none-win_arm64.whl", hash = "sha256:269302b31ade4cde6cf6f9dd58ea593773a37ed3f7b97e793c8594b262466b67"}, - {file = "ruff-0.1.14.tar.gz", hash = "sha256:ad3f8088b2dfd884820289a06ab718cde7d38b94972212cc4ba90d5fbc9955f3"}, + {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5fe8d54df166ecc24106db7dd6a68d44852d14eb0729ea4672bb4d96c320b7df"}, + {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6f0bfbb53c4b4de117ac4d6ddfd33aa5fc31beeaa21d23c45c6dd249faf9126f"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d432aec35bfc0d800d4f70eba26e23a352386be3a6cf157083d18f6f5881c8"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9405fa9ac0e97f35aaddf185a1be194a589424b8713e3b97b762336ec79ff807"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c66ec24fe36841636e814b8f90f572a8c0cb0e54d8b5c2d0e300d28a0d7bffec"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6f8ad828f01e8dd32cc58bc28375150171d198491fc901f6f98d2a39ba8e3ff5"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86811954eec63e9ea162af0ffa9f8d09088bab51b7438e8b6488b9401863c25e"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd4025ac5e87d9b80e1f300207eb2fd099ff8200fa2320d7dc066a3f4622dc6b"}, + {file = "ruff-0.1.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b17b93c02cdb6aeb696effecea1095ac93f3884a49a554a9afa76bb125c114c1"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ddb87643be40f034e97e97f5bc2ef7ce39de20e34608f3f829db727a93fb82c5"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:abf4822129ed3a5ce54383d5f0e964e7fef74a41e48eb1dfad404151efc130a2"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6c629cf64bacfd136c07c78ac10a54578ec9d1bd2a9d395efbee0935868bf852"}, + {file = "ruff-0.1.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1bab866aafb53da39c2cadfb8e1c4550ac5340bb40300083eb8967ba25481447"}, + {file = "ruff-0.1.15-py3-none-win32.whl", hash = "sha256:2417e1cb6e2068389b07e6fa74c306b2810fe3ee3476d5b8a96616633f40d14f"}, + {file = "ruff-0.1.15-py3-none-win_amd64.whl", hash = "sha256:3837ac73d869efc4182d9036b1405ef4c73d9b1f88da2413875e34e0d6919587"}, + {file = "ruff-0.1.15-py3-none-win_arm64.whl", hash = "sha256:9a933dfb1c14ec7a33cceb1e49ec4a16b51ce3c20fd42663198746efc0427360"}, + {file = "ruff-0.1.15.tar.gz", hash = "sha256:f6dfa8c1b21c913c326919056c390966648b680966febcb796cc9d1aaab8564e"}, ] [[package]] @@ -2282,13 +2282,13 @@ telegram = ["requests"] [[package]] name = "types-pytz" -version = "2023.3.1.1" +version = "2023.4.0.20240130" description = "Typing stubs for pytz" optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "types-pytz-2023.3.1.1.tar.gz", hash = "sha256:cc23d0192cd49c8f6bba44ee0c81e4586a8f30204970fc0894d209a6b08dab9a"}, - {file = "types_pytz-2023.3.1.1-py3-none-any.whl", hash = "sha256:1999a123a3dc0e39a2ef6d19f3f8584211de9e6a77fe7a0259f04a524e90a5cf"}, + {file = "types-pytz-2023.4.0.20240130.tar.gz", hash = "sha256:33676a90bf04b19f92c33eec8581136bea2f35ddd12759e579a624a006fd387a"}, + {file = "types_pytz-2023.4.0.20240130-py3-none-any.whl", hash = "sha256:6ce76a9f8fd22bd39b01a59c35bfa2db39b60d11a2f77145e97b730de7e64fe0"}, ] [[package]] @@ -2594,4 +2594,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "ccf3e64dcf9a2e22ee29c2a572fef6869e38bfcd6b096874d080adcd6f49612f" +content-hash = "6448dc0cf1e113399c3020790277989fc2c62b5ada6ff53ecd0f45da72d6953e" diff --git a/pyproject.toml b/pyproject.toml index 0dc009dda559..3669f7cba476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ docformatter = "^1.7.5" mypy = "^1.8.0" pandas-stubs = "^2.1.4" pre-commit = "^3.6.0" -ruff = "^0.1.14" +ruff = "^0.1.15" types-pytz = "^2023.3" types-requests = "^2.31" types-toml = "^0.10.2" From c93eee9322b4af7282cb326a8abbaa09cc80cc12 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 30 Jan 2024 20:17:52 +1100 Subject: [PATCH 32/42] Implement DatabentoLiveClient session handling --- .../live/databento/databento_subscriber.py | 27 +++-- nautilus_core/Cargo.lock | 1 + nautilus_core/adapters/Cargo.toml | 1 + .../adapters/src/databento/python/live.rs | 14 ++- nautilus_trader/adapters/databento/common.py | 11 +- nautilus_trader/adapters/databento/data.py | 106 ++++++++++++------ .../adapters/databento/providers.py | 18 +-- 7 files changed, 113 insertions(+), 65 deletions(-) diff --git a/examples/live/databento/databento_subscriber.py b/examples/live/databento/databento_subscriber.py index b6f89e7bd6b1..09481cd8c7fe 100644 --- a/examples/live/databento/databento_subscriber.py +++ b/examples/live/databento/databento_subscriber.py @@ -31,7 +31,6 @@ from nautilus_trader.model.data import OrderBookDeltas from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.data import TradeTick -from nautilus_trader.model.enums import BookType from nautilus_trader.model.identifiers import InstrumentId from nautilus_trader.model.identifiers import TraderId from nautilus_trader.trading.strategy import Strategy @@ -43,15 +42,15 @@ # For correct subscription operation, you must specify all instruments to be immediately # subscribed for as part of the data client configuration instrument_ids = [ - # InstrumentId.from_str("AAPL.XCHI"), - # InstrumentId.from_str("ESH4.GLBX"), + InstrumentId.from_str("AAPL.XCHI"), + InstrumentId.from_str("ESH4.GLBX"), InstrumentId.from_str("ESM4.GLBX"), ] # Configure the trading node config_node = TradingNodeConfig( trader_id=TraderId("TESTER-001"), - logging=LoggingConfig(log_level="INFO"), + logging=LoggingConfig(log_level="DEBUG"), exec_engine=LiveExecEngineConfig( reconciliation=False, # Not applicable inflight_check_interval_ms=0, # Not applicable @@ -131,7 +130,7 @@ def on_start(self) -> None: """ Actions to be performed when the strategy is started. - Here we specify the 'DATABENTO' client for subscriptions. + Here we specify the 'DATABENTO' client_id for subscriptions. """ for instrument_id in self.instrument_ids: @@ -140,15 +139,15 @@ def on_start(self) -> None: # book_type=BookType.L3_MBO, # client_id=DATABENTO_CLIENT_ID, # ) - self.subscribe_order_book_snapshots( - instrument_id=instrument_id, - book_type=BookType.L2_MBP, - depth=10, - client_id=DATABENTO_CLIENT_ID, - interval_ms=100, - ) - # self.subscribe_quote_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) - # self.subscribe_trade_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) + # self.subscribe_order_book_snapshots( + # instrument_id=instrument_id, + # book_type=BookType.L2_MBP, + # depth=10, + # client_id=DATABENTO_CLIENT_ID, + # interval_ms=100, + # ) + self.subscribe_quote_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) + self.subscribe_trade_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) # self.request_quote_ticks(instrument_id) # self.request_trade_ticks(instrument_id) # self.request_bars(BarType.from_str(f"{instrument_id}-1-MINUTE-LAST-EXTERNAL")) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index debcac8d0b2c..2369c4427151 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2440,6 +2440,7 @@ dependencies = [ "thiserror", "time", "tokio", + "tracing", "ustr", ] diff --git a/nautilus_core/adapters/Cargo.toml b/nautilus_core/adapters/Cargo.toml index e4ea251c3b9d..427fdbb08715 100644 --- a/nautilus_core/adapters/Cargo.toml +++ b/nautilus_core/adapters/Cargo.toml @@ -24,6 +24,7 @@ pyo3-asyncio = { workspace = true, optional = true } rand = { workspace = true } rust_decimal = { workspace = true } rust_decimal_macros = { workspace = true } +tracing = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 2b41b650e59e..b66a0adc4c4a 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -19,7 +19,7 @@ use std::sync::{Arc, OnceLock}; use anyhow::Result; use databento::live::Subscription; -use dbn::{PitSymbolMap, RType, Record, SymbolIndex}; +use dbn::{PitSymbolMap, RType, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; use log::{error, info}; use nautilus_core::python::to_pyruntime_err; @@ -59,6 +59,7 @@ impl DatabentoLiveClient { databento::LiveClient::builder() .key(&self.key)? .dataset(&self.dataset) + .upgrade_policy(VersionUpgradePolicy::Upgrade) .build() .await } @@ -109,6 +110,8 @@ impl DatabentoLiveClient { let arc_client = self.get_inner_client().map_err(to_pyruntime_err)?; pyo3_asyncio::tokio::future_into_py(py, async move { + // TODO: Attempt to obtain the mutex guard, if the client has already started then + // this will not be possible currently. let mut client = arc_client.lock().await; // TODO: This can be tidied up, conditionally calling `if let Some(start)` on @@ -130,6 +133,9 @@ impl DatabentoLiveClient { .build(), }; + // TODO: Temporary debug logging + println!("{:?}", subscription); + client .subscribe(&subscription) .await @@ -148,7 +154,9 @@ impl DatabentoLiveClient { let mut client = arc_client.lock().await; let mut symbol_map = PitSymbolMap::new(); - while let Some(record) = client.next_record().await.map_err(to_pyvalue_err)? { + client.start().await.map_err(to_pyruntime_err)?; + + while let Some(record) = client.next_record().await.map_err(to_pyruntime_err)? { let rtype = record.rtype().expect("Invalid `rtype`"); match rtype { RType::SymbolMapping => { @@ -163,7 +171,7 @@ impl DatabentoLiveClient { continue; } RType::System => { - println!("{record:?}"); // TODO: Just print stderr for now + println!("{record:?}"); // TODO: Just print stdout for now info!("{:?}", record); continue; } diff --git a/nautilus_trader/adapters/databento/common.py b/nautilus_trader/adapters/databento/common.py index 3f92c7cff3fc..a114b980a2e3 100644 --- a/nautilus_trader/adapters/databento/common.py +++ b/nautilus_trader/adapters/databento/common.py @@ -15,6 +15,7 @@ from pathlib import Path +from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.adapters.databento.types import DatabentoPublisher from nautilus_trader.core.correctness import PyCondition from nautilus_trader.model.data import BarType @@ -79,7 +80,7 @@ def nautilus_instrument_id_from_databento( return InstrumentId(Symbol(raw_symbol), Venue(publisher.venue)) -def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> str: +def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> DatabentoSchema: """ Return the Databento bar aggregate schema string for the given Nautilus `bar_type`. @@ -117,13 +118,13 @@ def databento_schema_from_nautilus_bar_type(bar_type: BarType) -> str: match bar_type.spec.aggregation: case BarAggregation.SECOND: - return "ohlcv-1s" + return DatabentoSchema.OHLCV_1S case BarAggregation.MINUTE: - return "ohlcv-1m" + return DatabentoSchema.OHLCV_1M case BarAggregation.HOUR: - return "ohlcv-1h" + return DatabentoSchema.OHLCV_1H case BarAggregation.DAY: - return "ohlcv-1d" + return DatabentoSchema.OHLCV_1D case _: raise ValueError( f"Invalid bar type '{bar_type}'. " diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 885a485fd692..c62e16a06042 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -27,6 +27,7 @@ from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH from nautilus_trader.adapters.databento.enums import DatabentoRecordFlags +from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.adapters.databento.providers import DatabentoInstrumentProvider from nautilus_trader.adapters.databento.types import Dataset @@ -259,7 +260,7 @@ def _get_live_client_mbo(self, dataset: Dataset) -> nautilus_pyo3.DatabentoLiveC return live_client - def _check_live_client_started( + async def _check_live_client_started( self, dataset: Dataset, live_client: nautilus_pyo3.DatabentoLiveClient, @@ -365,11 +366,15 @@ async def _subscribe_instrument(self, instrument_id: InstrumentId) -> None: try: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="definition", - symbols=instrument_id.symbol.value, + future = asyncio.ensure_future( + live_client.subscribe( + schema=DatabentoSchema.DEFINITION.value, + symbols=instrument_id.symbol.value, + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_instrument` was canceled while still pending.") @@ -380,12 +385,16 @@ async def _subscribe_parent_symbols( ) -> None: try: live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="definition", - symbols=",".join(sorted(parent_symbols)), - stype_in="parent", + future = asyncio.ensure_future( + live_client.subscribe( + schema=DatabentoSchema.DEFINITION.value, + symbols=",".join(sorted(parent_symbols)), + stype_in="parent", + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_parent_symbols` was canceled while still pending.") @@ -396,11 +405,15 @@ async def _subscribe_instrument_ids( ) -> None: try: live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="definition", - symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), + future = asyncio.ensure_future( + live_client.subscribe( + schema=DatabentoSchema.DEFINITION.value, + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_instrument_ids` was canceled while still pending.") @@ -469,13 +482,18 @@ async def _subscribe_order_book_deltas_batch( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_ids[0].venue) live_client = self._get_live_client_mbo(dataset) - await live_client.subscribe( - schema="mbo", - symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), - start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now + future = asyncio.ensure_future( + live_client.subscribe( + schema=DatabentoSchema.MBO.value, + symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), + start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now + ), ) + self._live_client_futures.add(future) + await future future = asyncio.ensure_future(live_client.start(self._handle_record)) self._live_client_futures.add(future) + await future except asyncio.CancelledError: self._log.warning( "`_subscribe_order_book_deltas_batch` was canceled while still pending.", @@ -493,9 +511,9 @@ async def _subscribe_order_book_snapshots( match depth: case 1: - schema = "mbp-1" + schema = DatabentoSchema.MBP_1.value case 10: - schema = "mbp-10" + schema = DatabentoSchema.MBP_10.value case _: self._log.error( f"Cannot subscribe for order book snapshots of depth {depth}, use either 1 or 10.", @@ -504,11 +522,15 @@ async def _subscribe_order_book_snapshots( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema=schema, - symbols=",".join(sorted([instrument_id.symbol.value])), + future = asyncio.ensure_future( + live_client.subscribe( + schema=schema, + symbols=",".join(sorted([instrument_id.symbol.value])), + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_order_book_snapshots` was canceled while still pending.") @@ -518,11 +540,15 @@ async def _subscribe_quote_ticks(self, instrument_id: InstrumentId) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="mbp-1", - symbols=",".join(sorted([instrument_id.symbol.value])), + future = asyncio.ensure_future( + live_client.subscribe( + schema=DatabentoSchema.MBP_1.value, + symbols=",".join(sorted([instrument_id.symbol.value])), + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_quote_ticks` was canceled while still pending.") @@ -535,11 +561,15 @@ async def _subscribe_trade_ticks(self, instrument_id: InstrumentId) -> None: dataset: Dataset = self._loader.get_dataset_for_venue(instrument_id.venue) live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema="trades", - symbols=instrument_id.symbol.value, + future = asyncio.ensure_future( + live_client.subscribe( + schema=DatabentoSchema.TRADES.value, + symbols=instrument_id.symbol.value, + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_trade_ticks` was canceled while still pending.") @@ -554,11 +584,15 @@ async def _subscribe_bars(self, bar_type: BarType) -> None: return live_client = self._get_live_client(dataset) - await live_client.subscribe( - schema=schema, - symbols=bar_type.instrument_id.symbol.value, + future = asyncio.ensure_future( + live_client.subscribe( + schema=schema.value, + symbols=bar_type.instrument_id.symbol.value, + ), ) - self._check_live_client_started(dataset, live_client) + self._live_client_futures.add(future) + await future + await self._check_live_client_started(dataset, live_client) except asyncio.CancelledError: self._log.warning("`_subscribe_bars` was canceled while still pending.") diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index 84c051df276e..bc5e063e1161 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -21,6 +21,7 @@ from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH +from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.common.component import LiveClock from nautilus_trader.common.providers import InstrumentProvider @@ -123,20 +124,23 @@ async def load_ids_async( pyo3_instruments = [] - async def receive_instruments(pyo3_instrument) -> None: - print(f"Received {pyo3_instrument}") + def receive_instruments(pyo3_instrument) -> None: pyo3_instruments.append(pyo3_instrument) - instrument_ids_to_decode.discard(instrument.id.value) - if not instrument_ids_to_decode: - raise asyncio.CancelledError("All instruments decoded") + instrument_ids_to_decode.discard(pyo3_instrument.id.value) + # TODO: Improve how to handle decode completion + # if not instrument_ids_to_decode: + # raise asyncio.CancelledError("All instruments decoded") await live_client.subscribe( - schema="definition", + schema=DatabentoSchema.DEFINITION.value, symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), start=0, # From start of current session (latest definition) ) - await asyncio.wait_for(live_client.start(callback=receive_instruments), timeout=5.0) + try: + await asyncio.wait_for(live_client.start(callback=receive_instruments), timeout=5.0) + except asyncio.CancelledError: + pass # Expected on decode completion, continue instruments = instruments_from_pyo3(pyo3_instruments) From 6f940830644752ed4cd10e88f132ea9c850ed0aa Mon Sep 17 00:00:00 2001 From: Sunlei Date: Wed, 31 Jan 2024 03:10:00 +0800 Subject: [PATCH 33/42] Fix unused import psutil --- nautilus_trader/common/component.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/nautilus_trader/common/component.pyx b/nautilus_trader/common/component.pyx index 16600e0bf30c..4b21f294bc18 100644 --- a/nautilus_trader/common/component.pyx +++ b/nautilus_trader/common/component.pyx @@ -30,7 +30,6 @@ import cython import msgspec import numpy as np import pandas as pd -import psutil import pyarrow import pytz From 07cc94eab098c874de8927481721c33a596e9bee Mon Sep 17 00:00:00 2001 From: Sunlei Date: Wed, 31 Jan 2024 04:32:03 +0800 Subject: [PATCH 34/42] Fix Binance non-existing method name --- nautilus_trader/adapters/binance/spot/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_trader/adapters/binance/spot/execution.py b/nautilus_trader/adapters/binance/spot/execution.py index 9d4b676022de..2595a9ce2af8 100644 --- a/nautilus_trader/adapters/binance/spot/execution.py +++ b/nautilus_trader/adapters/binance/spot/execution.py @@ -224,4 +224,4 @@ def _handle_list_status(self, raw: bytes) -> None: self._log.warning("List status (OCO) received.") # Implement def _handle_balance_update(self, raw: bytes) -> None: - self.create_task(self._update_account_state_async()) + self.create_task(self._update_account_state()) From 51c05918a72cb9cab3eb89237a30cc9cdf8bdd8c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 31 Jan 2024 18:18:37 +1100 Subject: [PATCH 35/42] Add poetry sync to install-debug make target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 572377c9d732..0dc914ffa1d3 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ install: .PHONY: install-debug install-debug: - BUILD_MODE=debug poetry install --with dev,test --all-extras + BUILD_MODE=debug poetry install --with dev,test --all-extras --sync .PHONY: install-just-deps install-just-deps: From 5399de0e05d42d7b51867c9c4df19aca2aa2814c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 31 Jan 2024 18:19:42 +1100 Subject: [PATCH 36/42] Update copyright headers --- nautilus_core/common/src/python/logging.rs | 2 +- nautilus_core/model/src/accounting/margin.rs | 2 +- nautilus_core/model/src/accounting/mod.rs | 2 +- nautilus_core/model/src/accounting/stubs.rs | 2 +- nautilus_core/model/src/python/accounting/margin.rs | 2 +- nautilus_core/model/src/python/accounting/mod.rs | 2 +- .../adapters/interactive_brokers/client/test_client.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nautilus_core/common/src/python/logging.rs b/nautilus_core/common/src/python/logging.rs index b46632ea42cd..b0b7a95ec587 100644 --- a/nautilus_core/common/src/python/logging.rs +++ b/nautilus_core/common/src/python/logging.rs @@ -1,5 +1,5 @@ // ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. // https://nautechsystems.io // // Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); diff --git a/nautilus_core/model/src/accounting/margin.rs b/nautilus_core/model/src/accounting/margin.rs index 490d9deb4d86..5ed625b4a788 100644 --- a/nautilus_core/model/src/accounting/margin.rs +++ b/nautilus_core/model/src/accounting/margin.rs @@ -1,5 +1,5 @@ // ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. // https://nautechsystems.io // // Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); diff --git a/nautilus_core/model/src/accounting/mod.rs b/nautilus_core/model/src/accounting/mod.rs index 76725f723a1f..88f1f6c55a64 100644 --- a/nautilus_core/model/src/accounting/mod.rs +++ b/nautilus_core/model/src/accounting/mod.rs @@ -1,5 +1,5 @@ // ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. // https://nautechsystems.io // // Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); diff --git a/nautilus_core/model/src/accounting/stubs.rs b/nautilus_core/model/src/accounting/stubs.rs index 6677ae8f9d27..a27192575371 100644 --- a/nautilus_core/model/src/accounting/stubs.rs +++ b/nautilus_core/model/src/accounting/stubs.rs @@ -1,5 +1,5 @@ // ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. // https://nautechsystems.io // // Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); diff --git a/nautilus_core/model/src/python/accounting/margin.rs b/nautilus_core/model/src/python/accounting/margin.rs index 78b3716f2f79..eaaf4ddc8940 100644 --- a/nautilus_core/model/src/python/accounting/margin.rs +++ b/nautilus_core/model/src/python/accounting/margin.rs @@ -1,5 +1,5 @@ // ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. // https://nautechsystems.io // // Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); diff --git a/nautilus_core/model/src/python/accounting/mod.rs b/nautilus_core/model/src/python/accounting/mod.rs index 1b0a0a8bb021..d0c09c95d89c 100644 --- a/nautilus_core/model/src/python/accounting/mod.rs +++ b/nautilus_core/model/src/python/accounting/mod.rs @@ -1,5 +1,5 @@ // ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. // https://nautechsystems.io // // Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client.py index 94bb76e06b32..842cdd345fa5 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------------------------- -# Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. # https://nautechsystems.io # # Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); From 0533208c7a790efe6f3dd2cb34982b4f79c7e223 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 31 Jan 2024 19:37:07 +1100 Subject: [PATCH 37/42] Refine DatabentoLiveClient session handling --- .../live/databento/databento_subscriber.py | 2 +- .../adapters/src/databento/python/live.rs | 30 +++++++++++++------ .../adapters/databento/providers.py | 3 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/examples/live/databento/databento_subscriber.py b/examples/live/databento/databento_subscriber.py index 09481cd8c7fe..1969c32909af 100644 --- a/examples/live/databento/databento_subscriber.py +++ b/examples/live/databento/databento_subscriber.py @@ -50,7 +50,7 @@ # Configure the trading node config_node = TradingNodeConfig( trader_id=TraderId("TESTER-001"), - logging=LoggingConfig(log_level="DEBUG"), + logging=LoggingConfig(log_level="INFO"), exec_engine=LiveExecEngineConfig( reconciliation=False, # Not applicable inflight_check_interval_ms=0, # Not applicable diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index b66a0adc4c4a..d6df3c633137 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -34,6 +34,7 @@ use nautilus_model::identifiers::venue::Venue; use pyo3::prelude::*; use time::OffsetDateTime; use tokio::sync::Mutex; +use tokio::time::{timeout, Duration}; use crate::databento::parsing::{parse_instrument_def_msg, parse_record}; use crate::databento::types::{DatabentoPublisher, PublisherId}; @@ -110,8 +111,6 @@ impl DatabentoLiveClient { let arc_client = self.get_inner_client().map_err(to_pyruntime_err)?; pyo3_asyncio::tokio::future_into_py(py, async move { - // TODO: Attempt to obtain the mutex guard, if the client has already started then - // this will not be possible currently. let mut client = arc_client.lock().await; // TODO: This can be tidied up, conditionally calling `if let Some(start)` on @@ -134,7 +133,7 @@ impl DatabentoLiveClient { }; // TODO: Temporary debug logging - println!("{:?}", subscription); + // println!("{:?}", subscription); client .subscribe(&subscription) @@ -154,26 +153,39 @@ impl DatabentoLiveClient { let mut client = arc_client.lock().await; let mut symbol_map = PitSymbolMap::new(); + let timeout_duration = Duration::from_millis(10); client.start().await.map_err(to_pyruntime_err)?; - while let Some(record) = client.next_record().await.map_err(to_pyruntime_err)? { + loop { + drop(client); + client = arc_client.lock().await; + + let result = timeout(timeout_duration, client.next_record()).await; + let record = match result { + Ok(Ok(Some(record))) => record, + Ok(Ok(None)) => break, // Session ended normally + Ok(Err(e)) => { + // Fail session entirely for now + return Err(to_pyruntime_err(e)); + } + Err(_) => continue, // Timeout + }; + let rtype = record.rtype().expect("Invalid `rtype`"); + match rtype { RType::SymbolMapping => { symbol_map.on_record(record).unwrap_or_else(|_| { panic!("Error updating `symbol_map` with {record:?}") }); - continue; } RType::Error => { eprintln!("{record:?}"); // TODO: Just print stderr for now error!("{:?}", record); - continue; } RType::System => { println!("{record:?}"); // TODO: Just print stdout for now info!("{:?}", record); - continue; } RType::InstrumentDef => { let msg = record @@ -186,7 +198,7 @@ impl DatabentoLiveClient { match result { Ok(instrument) => { - // TODO: Improve the efficiency of this constant GIL aquisition + // TODO: Optimize this by reducing the frequency of acquiring the GIL if possible Python::with_gil(|py| { let py_obj = convert_instrument_to_pyobject(py, instrument).unwrap(); @@ -217,7 +229,7 @@ impl DatabentoLiveClient { parse_record(&record, rtype, instrument_id, 2, Some(ts_init)) .map_err(to_pyvalue_err)?; - // TODO: Improve the efficiency of this constant GIL aquisition + // TODO: Optimize this by reducing the frequency of acquiring the GIL if possible Python::with_gil(|py| { let py_obj = match data { Data::Delta(delta) => delta.into_py(py), diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index bc5e063e1161..8a00493e78c1 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -15,6 +15,7 @@ import asyncio import datetime as dt +from typing import Any import pandas as pd import pytz @@ -124,7 +125,7 @@ async def load_ids_async( pyo3_instruments = [] - def receive_instruments(pyo3_instrument) -> None: + def receive_instruments(pyo3_instrument: Any) -> None: pyo3_instruments.append(pyo3_instrument) instrument_ids_to_decode.discard(pyo3_instrument.id.value) # TODO: Improve how to handle decode completion From e0309f97af1bce683df9df43169b9b61b530c18d Mon Sep 17 00:00:00 2001 From: Filip Macek Date: Thu, 1 Feb 2024 05:14:42 +0100 Subject: [PATCH 38/42] Implement Margin account in Rust (#1482) --- nautilus_core/Cargo.lock | 15 + nautilus_core/Cargo.toml | 1 + nautilus_core/accounting/Cargo.toml | 36 ++ nautilus_core/accounting/src/account/base.rs | 238 +++++++ nautilus_core/accounting/src/account/cash.rs | 581 ++++++++++++++++++ .../src/account}/margin.rs | 234 +++++-- nautilus_core/accounting/src/account/mod.rs | 75 +++ .../src/account}/stubs.rs | 23 +- .../mod.rs => accounting/src/lib.rs} | 6 +- nautilus_core/accounting/src/position.rs | 126 ++++ nautilus_core/accounting/src/python/cash.rs | 361 +++++++++++ .../src/python}/margin.rs | 33 +- .../src/python}/mod.rs | 14 +- .../accounting/src/python/position.rs | 54 ++ nautilus_core/common/Cargo.toml | 3 +- nautilus_core/common/src/lib.rs | 2 +- nautilus_core/model/src/enums.rs | 1 + .../model/src/events/account/state.rs | 8 +- .../model/src/events/account/stubs.rs | 83 ++- nautilus_core/model/src/instruments/stubs.rs | 5 + nautilus_core/model/src/lib.rs | 1 - nautilus_core/model/src/orders/stubs.rs | 4 +- .../model/src/python/events/account/state.rs | 21 +- nautilus_core/model/src/python/mod.rs | 4 - .../model/src/python/orders/market.rs | 22 + nautilus_core/pyo3/Cargo.toml | 1 + nautilus_core/pyo3/src/lib.rs | 6 + nautilus_trader/core/nautilus_pyo3.pyi | 193 +++++- .../test_kit/rust/accounting_pyo3.py | 22 + nautilus_trader/test_kit/rust/events_pyo3.py | 143 +++++ tests/unit_tests/accounting/test_cash_pyo3.py | 244 ++++++++ .../unit_tests/accounting/test_margin_pyo3.py | 3 +- 32 files changed, 2450 insertions(+), 113 deletions(-) create mode 100644 nautilus_core/accounting/Cargo.toml create mode 100644 nautilus_core/accounting/src/account/base.rs create mode 100644 nautilus_core/accounting/src/account/cash.rs rename nautilus_core/{model/src/accounting => accounting/src/account}/margin.rs (72%) create mode 100644 nautilus_core/accounting/src/account/mod.rs rename nautilus_core/{model/src/accounting => accounting/src/account}/stubs.rs (61%) rename nautilus_core/{model/src/python/accounting/mod.rs => accounting/src/lib.rs} (91%) create mode 100644 nautilus_core/accounting/src/position.rs create mode 100644 nautilus_core/accounting/src/python/cash.rs rename nautilus_core/{model/src/python/accounting => accounting/src/python}/margin.rs (90%) rename nautilus_core/{model/src/accounting => accounting/src/python}/mod.rs (73%) create mode 100644 nautilus_core/accounting/src/python/position.rs create mode 100644 tests/unit_tests/accounting/test_cash_pyo3.py diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 2369c4427151..92be58006640 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2413,6 +2413,20 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nautilus-accounting" +version = "0.17.0" +dependencies = [ + "anyhow", + "cbindgen", + "nautilus-common", + "nautilus-core", + "nautilus-model", + "pyo3", + "rstest", + "rust_decimal", +] + [[package]] name = "nautilus-adapters" version = "0.17.0" @@ -2609,6 +2623,7 @@ dependencies = [ name = "nautilus-pyo3" version = "0.17.0" dependencies = [ + "nautilus-accounting", "nautilus-adapters", "nautilus-common", "nautilus-core", diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 2dc8cb91514f..4d33d3ce348d 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -12,6 +12,7 @@ members = [ "network/tokio-tungstenite", "persistence", "pyo3", + "accounting" ] [workspace.package] diff --git a/nautilus_core/accounting/Cargo.toml b/nautilus_core/accounting/Cargo.toml new file mode 100644 index 000000000000..96d17bf84466 --- /dev/null +++ b/nautilus_core/accounting/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "nautilus-accounting" +version.workspace = true +edition.workspace = true +authors.workspace = true +description.workspace = true +documentation.workspace = true + +[lib] +name = "nautilus_accounting" +crate-type = ["rlib", "cdylib"] + +[dependencies] +nautilus-common = { path = "../common", features = ["stubs"] } +nautilus-model = { path = "../model", features = ["stubs"]} +nautilus-core = { path = "../core" } +pyo3 = { workspace = true, optional = true } +anyhow = { workspace = true } +rust_decimal = { workspace = true } + +[dev-dependencies] +rstest.workspace = true + + +[features] +extension-module = [ + "pyo3/extension-module", + "nautilus-core/extension-module", + "nautilus-common/extension-module", +] +python = ["pyo3"] +default = ["python"] + + +[build-dependencies] +cbindgen = { workspace = true, optional = true } \ No newline at end of file diff --git a/nautilus_core/accounting/src/account/base.rs b/nautilus_core/accounting/src/account/base.rs new file mode 100644 index 000000000000..db3a06bb5f56 --- /dev/null +++ b/nautilus_core/accounting/src/account/base.rs @@ -0,0 +1,238 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use crate::position::Position; +use anyhow::Result; +use nautilus_model::enums::{AccountType, LiquiditySide, OrderSide}; +use nautilus_model::events::account::state::AccountState; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::identifiers::account_id::AccountId; +use nautilus_model::instruments::Instrument; +use nautilus_model::types::balance::AccountBalance; +use nautilus_model::types::currency::Currency; +use nautilus_model::types::money::Money; +use nautilus_model::types::price::Price; +use nautilus_model::types::quantity::Quantity; +use pyo3::prelude::*; +use rust_decimal::prelude::ToPrimitive; +use std::collections::HashMap; + +#[derive(Debug)] +#[cfg_attr( + feature = "python", + pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") +)] +pub struct BaseAccount { + #[pyo3(get)] + pub id: AccountId, + pub account_type: AccountType, + pub base_currency: Option, + pub calculate_account_state: bool, + pub events: Vec, + pub commissions: HashMap, + pub balances: HashMap, + pub balances_starting: HashMap, +} + +impl BaseAccount { + pub fn new(event: AccountState, calculate_account_state: bool) -> Result { + let mut balances_starting: HashMap = HashMap::new(); + let mut balances: HashMap = HashMap::new(); + event.balances.iter().for_each(|balance| { + balances_starting.insert(balance.currency, balance.total); + balances.insert(balance.currency, *balance); + }); + Ok(Self { + id: event.account_id, + account_type: event.account_type, + base_currency: event.base_currency, + calculate_account_state, + events: vec![event], + commissions: HashMap::new(), + balances, + balances_starting, + }) + } + + pub fn base_balance_total(&self, currency: Option) -> Option { + let currency = currency + .or(self.base_currency) + .expect("Currency must be specified"); + let account_balance = self.balances.get(¤cy); + account_balance.map(|balance| balance.total) + } + + pub fn base_balances_total(&self) -> HashMap { + self.balances + .iter() + .map(|(currency, balance)| (*currency, balance.total)) + .collect() + } + + pub fn base_balance_free(&self, currency: Option) -> Option { + let currency = currency + .or(self.base_currency) + .expect("Currency must be specified"); + let account_balance = self.balances.get(¤cy); + account_balance.map(|balance| balance.free) + } + + pub fn base_balances_free(&self) -> HashMap { + self.balances + .iter() + .map(|(currency, balance)| (*currency, balance.free)) + .collect() + } + + pub fn base_balance_locked(&self, currency: Option) -> Option { + let currency = currency + .or(self.base_currency) + .expect("Currency must be specified"); + let account_balance = self.balances.get(¤cy); + account_balance.map(|balance| balance.locked) + } + + pub fn base_balances_locked(&self) -> HashMap { + self.balances + .iter() + .map(|(currency, balance)| (*currency, balance.locked)) + .collect() + } + + pub fn base_last_event(&self) -> Option { + self.events.last().cloned() + } + pub fn update_balances(&mut self, balances: Vec) { + balances.into_iter().for_each(|balance| { + // clone real balance without reference + if balance.total.raw < 0 { + // TODO raise AccountBalanceNegative event + panic!("Cannot update balances with total less than 0.0") + } else { + // clear asset balance + self.balances.insert(balance.currency, balance); + } + }); + } + pub fn base_apply(&mut self, event: AccountState) { + self.update_balances(event.balances.clone()); + self.events.push(event.clone()); + } + + pub fn base_calculate_balance_locked( + &mut self, + instrument: T, + side: OrderSide, + quantity: Quantity, + price: Price, + use_quote_for_inverse: Option, + ) -> Result { + let base_currency = instrument + .base_currency() + .unwrap_or(instrument.quote_currency()); + let quote_currency = instrument.quote_currency(); + let notional: f64 = match side { + OrderSide::Buy => instrument + .calculate_notional_value(quantity, price, use_quote_for_inverse) + .as_f64(), + OrderSide::Sell => quantity.as_f64(), + _ => panic!("Invalid order side in base_calculate_balance_locked"), + }; + // add expected commission + let taker_fee = instrument.taker_fee().to_f64().unwrap(); + let locked: f64 = notional + (notional * taker_fee * 2.0); + + // handle inverse + if instrument.is_inverse() && !use_quote_for_inverse.unwrap_or(false) { + Ok(Money::new(locked, *base_currency).unwrap()) + } else if side == OrderSide::Buy { + Ok(Money::new(locked, *quote_currency).unwrap()) + } else if side == OrderSide::Sell { + Ok(Money::new(locked, *base_currency).unwrap()) + } else { + panic!("Invalid order side in base_calculate_balance_locked") + } + } + + pub fn base_calculate_pnls( + &self, + instrument: T, + fill: OrderFilled, + position: Option, + ) -> Result> { + let mut pnls: HashMap = HashMap::new(); + let quote_currency = instrument.quote_currency(); + let base_currency = instrument.base_currency(); + + let fill_px = fill.last_px.as_f64(); + let fill_qty = position.map_or(fill.last_qty.as_f64(), |pos| { + pos.quantity.as_f64().min(fill.last_qty.as_f64()) + }); + if fill.order_side == OrderSide::Buy { + if let (Some(base_currency_value), None) = (base_currency, self.base_currency) { + pnls.insert( + *base_currency_value, + Money::new(fill_qty, *base_currency_value).unwrap(), + ); + } + pnls.insert( + *quote_currency, + Money::new(-(fill_qty * fill_px), *quote_currency).unwrap(), + ); + } else if fill.order_side == OrderSide::Sell { + if let (Some(base_currency_value), None) = (base_currency, self.base_currency) { + pnls.insert( + *base_currency_value, + Money::new(-fill_qty, *base_currency_value).unwrap(), + ); + } + pnls.insert( + *quote_currency, + Money::new(fill_qty * fill_px, *quote_currency).unwrap(), + ); + } else { + panic!("Invalid order side in base_calculate_pnls") + } + Ok(pnls.into_values().collect()) + } + + pub fn base_calculate_commission( + &self, + instrument: T, + last_qty: Quantity, + last_px: Price, + liquidity_side: LiquiditySide, + use_quote_for_inverse: Option, + ) -> Result { + if liquidity_side == LiquiditySide::NoLiquiditySide { + panic!("Invalid liquidity side") + } + let notional = instrument + .calculate_notional_value(last_qty, last_px, use_quote_for_inverse) + .as_f64(); + let commission = if liquidity_side == LiquiditySide::Maker { + notional * instrument.maker_fee().to_f64().unwrap() + } else if liquidity_side == LiquiditySide::Taker { + notional * instrument.taker_fee().to_f64().unwrap() + } else { + panic!("Invalid liquid side {}", liquidity_side) + }; + if instrument.is_inverse() && !use_quote_for_inverse.unwrap_or(false) { + Ok(Money::new(commission, *instrument.base_currency().unwrap()).unwrap()) + } else { + Ok(Money::new(commission, *instrument.quote_currency()).unwrap()) + } + } +} diff --git a/nautilus_core/accounting/src/account/cash.rs b/nautilus_core/accounting/src/account/cash.rs new file mode 100644 index 000000000000..e92afb972674 --- /dev/null +++ b/nautilus_core/accounting/src/account/cash.rs @@ -0,0 +1,581 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use crate::account::base::BaseAccount; +use crate::account::Account; +use crate::position::Position; +use anyhow::Result; +use nautilus_model::enums::{AccountType, LiquiditySide, OrderSide}; +use nautilus_model::events::account::state::AccountState; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::instruments::Instrument; +use nautilus_model::types::balance::AccountBalance; +use nautilus_model::types::currency::Currency; +use nautilus_model::types::money::Money; +use nautilus_model::types::price::Price; +use nautilus_model::types::quantity::Quantity; +use pyo3::prelude::*; +use std::collections::HashMap; +use std::fmt::Display; +use std::ops::{Deref, DerefMut}; + +#[derive(Debug)] +#[cfg_attr( + feature = "python", + pyclass(module = "nautilus_trader.core.nautilus_pyo3.accounting") +)] +pub struct CashAccount { + pub base: BaseAccount, +} + +impl CashAccount { + pub fn new(event: AccountState, calculate_account_state: bool) -> Result { + Ok(Self { + base: BaseAccount::new(event, calculate_account_state)?, + }) + } + + pub fn is_cash_account(&self) -> bool { + self.account_type == AccountType::Cash + } + pub fn is_margin_account(&self) -> bool { + self.account_type == AccountType::Margin + } + + pub fn is_unleveraged(&self) -> bool { + false + } +} + +impl Account for CashAccount { + fn balance_total(&self, currency: Option) -> Option { + self.base_balance_total(currency) + } + fn balances_total(&self) -> HashMap { + self.base_balances_total() + } + + fn balance_free(&self, currency: Option) -> Option { + self.base_balance_free(currency) + } + + fn balances_free(&self) -> HashMap { + self.base_balances_free() + } + fn balance_locked(&self, currency: Option) -> Option { + self.base_balance_locked(currency) + } + fn balances_locked(&self) -> HashMap { + self.base_balances_locked() + } + fn last_event(&self) -> Option { + self.base_last_event() + } + fn events(&self) -> Vec { + self.events.clone() + } + fn event_count(&self) -> usize { + self.events.len() + } + fn currencies(&self) -> Vec { + self.balances.keys().copied().collect() + } + fn starting_balances(&self) -> HashMap { + self.balances_starting.clone() + } + fn balances(&self) -> HashMap { + self.balances.clone() + } + fn apply(&mut self, event: AccountState) { + self.base_apply(event) + } + + fn calculate_balance_locked( + &mut self, + instrument: T, + side: OrderSide, + quantity: Quantity, + price: Price, + use_quote_for_inverse: Option, + ) -> Result { + self.base_calculate_balance_locked(instrument, side, quantity, price, use_quote_for_inverse) + } + fn calculate_pnls( + &self, + instrument: T, + fill: OrderFilled, + position: Option, + ) -> Result> { + self.base_calculate_pnls(instrument, fill, position) + } + fn calculate_commission( + &self, + instrument: T, + last_qty: Quantity, + last_px: Price, + liquidity_side: LiquiditySide, + use_quote_for_inverse: Option, + ) -> Result { + self.base_calculate_commission( + instrument, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + } +} + +impl Deref for CashAccount { + type Target = BaseAccount; + + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl DerefMut for CashAccount { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.base + } +} + +impl PartialEq for CashAccount { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for CashAccount {} + +impl Display for CashAccount { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "CashAccount(id={}, type={}, base={})", + self.id, + self.account_type, + self.base_currency + .map(|base_currency| format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), + ) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////////////// +#[cfg(test)] +mod tests { + use crate::account::cash::CashAccount; + use crate::account::stubs::*; + use crate::account::Account; + use crate::position::Position; + use nautilus_common::factories::OrderFactory; + use nautilus_common::stubs::*; + use nautilus_model::enums::{AccountType, LiquiditySide, OrderSide}; + use nautilus_model::events::account::state::AccountState; + use nautilus_model::events::account::stubs::*; + use nautilus_model::identifiers::account_id::AccountId; + use nautilus_model::identifiers::position_id::PositionId; + use nautilus_model::identifiers::strategy_id::StrategyId; + use nautilus_model::instruments::crypto_perpetual::CryptoPerpetual; + use nautilus_model::instruments::currency_pair::CurrencyPair; + use nautilus_model::instruments::equity::Equity; + use nautilus_model::instruments::stubs::*; + use nautilus_model::orders::market::MarketOrder; + use nautilus_model::orders::stubs::TestOrderEventStubs; + use nautilus_model::types::currency::Currency; + use nautilus_model::types::money::Money; + use nautilus_model::types::price::Price; + use nautilus_model::types::quantity::Quantity; + use rstest::rstest; + use std::collections::HashMap; + use std::collections::HashSet; + + #[rstest] + fn test_display(cash_account: CashAccount) { + assert_eq!( + format!("{}", cash_account), + "CashAccount(id=SIM-001, type=CASH, base=USD)" + ) + } + + #[rstest] + fn test_instantiate_single_asset_cash_account( + cash_account: CashAccount, + cash_account_state: AccountState, + ) { + assert_eq!(cash_account.id, AccountId::from("SIM-001")); + assert_eq!(cash_account.account_type, AccountType::Cash); + assert_eq!(cash_account.base_currency, Some(Currency::from("USD"))); + assert_eq!(cash_account.last_event(), Some(cash_account_state.clone())); + assert_eq!(cash_account.events(), vec![cash_account_state.clone()]); + assert_eq!(cash_account.event_count(), 1); + assert_eq!( + cash_account.balance_total(None), + Some(Money::from("1525000 USD")) + ); + assert_eq!( + cash_account.balance_free(None), + Some(Money::from("1500000 USD")) + ); + assert_eq!( + cash_account.balance_locked(None), + Some(Money::from("25000 USD")) + ); + let mut balances_total_expected = HashMap::new(); + balances_total_expected.insert(Currency::from("USD"), Money::from("1525000 USD")); + assert_eq!(cash_account.balances_total(), balances_total_expected); + let mut balances_free_expected = HashMap::new(); + balances_free_expected.insert(Currency::from("USD"), Money::from("1500000 USD")); + assert_eq!(cash_account.balances_free(), balances_free_expected); + let mut balances_locked_expected = HashMap::new(); + balances_locked_expected.insert(Currency::from("USD"), Money::from("25000 USD")); + assert_eq!(cash_account.balances_locked(), balances_locked_expected); + } + + #[rstest] + fn test_instantiate_multi_asset_cash_account( + cash_account_multi: CashAccount, + cash_account_state_multi: AccountState, + ) { + assert_eq!(cash_account_multi.id, AccountId::from("SIM-001")); + assert_eq!(cash_account_multi.account_type, AccountType::Cash); + assert_eq!( + cash_account_multi.last_event(), + Some(cash_account_state_multi.clone()) + ); + assert_eq!(cash_account_state_multi.base_currency, None); + assert_eq!( + cash_account_multi.events(), + vec![cash_account_state_multi.clone()] + ); + assert_eq!(cash_account_multi.event_count(), 1); + assert_eq!( + cash_account_multi.balance_total(Some(Currency::BTC())), + Some(Money::from("10 BTC")) + ); + assert_eq!( + cash_account_multi.balance_total(Some(Currency::ETH())), + Some(Money::from("20 ETH")) + ); + assert_eq!( + cash_account_multi.balance_free(Some(Currency::BTC())), + Some(Money::from("10 BTC")) + ); + assert_eq!( + cash_account_multi.balance_free(Some(Currency::ETH())), + Some(Money::from("20 ETH")) + ); + assert_eq!( + cash_account_multi.balance_locked(Some(Currency::BTC())), + Some(Money::from("0 BTC")) + ); + assert_eq!( + cash_account_multi.balance_locked(Some(Currency::ETH())), + Some(Money::from("0 ETH")) + ); + let mut balances_total_expected = HashMap::new(); + balances_total_expected.insert(Currency::from("BTC"), Money::from("10 BTC")); + balances_total_expected.insert(Currency::from("ETH"), Money::from("20 ETH")); + assert_eq!(cash_account_multi.balances_total(), balances_total_expected); + let mut balances_free_expected = HashMap::new(); + balances_free_expected.insert(Currency::from("BTC"), Money::from("10 BTC")); + balances_free_expected.insert(Currency::from("ETH"), Money::from("20 ETH")); + assert_eq!(cash_account_multi.balances_free(), balances_free_expected); + let mut balances_locked_expected = HashMap::new(); + balances_locked_expected.insert(Currency::from("BTC"), Money::from("0 BTC")); + balances_locked_expected.insert(Currency::from("ETH"), Money::from("0 ETH")); + assert_eq!( + cash_account_multi.balances_locked(), + balances_locked_expected + ); + } + + #[rstest] + fn test_apply_given_new_state_event_updates_correctly( + mut cash_account_multi: CashAccount, + cash_account_state_multi: AccountState, + cash_account_state_multi_changed_btc: AccountState, + ) { + // apply second account event + cash_account_multi.apply(cash_account_state_multi_changed_btc.clone()); + assert_eq!( + cash_account_multi.last_event(), + Some(cash_account_state_multi_changed_btc.clone()) + ); + assert_eq!( + cash_account_multi.events, + vec![ + cash_account_state_multi.clone(), + cash_account_state_multi_changed_btc.clone() + ] + ); + assert_eq!(cash_account_multi.event_count(), 2); + assert_eq!( + cash_account_multi.balance_total(Some(Currency::BTC())), + Some(Money::from("9 BTC")) + ); + assert_eq!( + cash_account_multi.balance_free(Some(Currency::BTC())), + Some(Money::from("8.5 BTC")) + ); + assert_eq!( + cash_account_multi.balance_locked(Some(Currency::BTC())), + Some(Money::from("0.5 BTC")) + ); + assert_eq!( + cash_account_multi.balance_total(Some(Currency::ETH())), + Some(Money::from("20 ETH")) + ); + assert_eq!( + cash_account_multi.balance_free(Some(Currency::ETH())), + Some(Money::from("20 ETH")) + ); + assert_eq!( + cash_account_multi.balance_locked(Some(Currency::ETH())), + Some(Money::from("0 ETH")) + ); + } + + #[rstest] + fn test_calculate_balance_locked_buy( + mut cash_account_million_usd: CashAccount, + audusd_sim: CurrencyPair, + ) { + let balance_locked = cash_account_million_usd + .calculate_balance_locked( + audusd_sim, + OrderSide::Buy, + Quantity::from("1000000"), + Price::from("0.8"), + None, + ) + .unwrap(); + assert_eq!(balance_locked, Money::from("800032 USD")) + } + + #[rstest] + fn test_calculate_balance_locked_sell( + mut cash_account_million_usd: CashAccount, + audusd_sim: CurrencyPair, + ) { + let balance_locked = cash_account_million_usd + .calculate_balance_locked( + audusd_sim, + OrderSide::Sell, + Quantity::from("1000000"), + Price::from("0.8"), + None, + ) + .unwrap(); + assert_eq!(balance_locked, Money::from("1000040 AUD")) + } + + #[rstest] + fn test_calculate_balance_locked_sell_no_base_currency( + mut cash_account_million_usd: CashAccount, + equity_aapl: Equity, + ) { + let balance_locked = cash_account_million_usd + .calculate_balance_locked( + equity_aapl, + OrderSide::Sell, + Quantity::from("100"), + Price::from("1500.0"), + None, + ) + .unwrap(); + assert_eq!(balance_locked, Money::from("100 USD")) + } + + #[rstest] + fn test_calculate_pnls_for_single_currency_cash_account( + cash_account_million_usd: CashAccount, + mut order_factory: OrderFactory, + audusd_sim: CurrencyPair, + ) { + let order = order_factory.market( + audusd_sim.id, + OrderSide::Buy, + Quantity::from("1000000"), + None, + None, + None, + None, + None, + None, + ); + let fill = TestOrderEventStubs::order_filled::( + order, + audusd_sim, + Some(StrategyId::new("S-001").unwrap()), + None, + Some(PositionId::new("P-123456").unwrap()), + Some(Price::from("0.8")), + None, + None, + None, + ); + let position = Position::new(audusd_sim, fill).unwrap(); + let pnls = cash_account_million_usd + .calculate_pnls(audusd_sim, fill, Some(position)) + .unwrap(); + assert_eq!(pnls, vec![Money::from("-800000 USD")]) + } + + #[rstest] + fn test_calculate_pnls_for_multi_currency_cash_account_btcusdt( + cash_account_multi: CashAccount, + mut order_factory: OrderFactory, + currency_pair_btcusdt: CurrencyPair, + ) { + let order1 = order_factory.market( + currency_pair_btcusdt.id, + OrderSide::Sell, + Quantity::from("0.5"), + None, + None, + None, + None, + None, + None, + ); + let fill1 = TestOrderEventStubs::order_filled::( + order1, + currency_pair_btcusdt, + Some(StrategyId::new("S-001").unwrap()), + None, + Some(PositionId::new("P-123456").unwrap()), + Some(Price::from("45500.00")), + None, + None, + None, + ); + let position = Position::new(currency_pair_btcusdt, fill1).unwrap(); + let result1 = cash_account_multi + .calculate_pnls(currency_pair_btcusdt, fill1, Some(position.clone())) + .unwrap(); + let order2 = order_factory.market( + currency_pair_btcusdt.id, + OrderSide::Buy, + Quantity::from("0.5"), + None, + None, + None, + None, + None, + None, + ); + let fill2 = TestOrderEventStubs::order_filled::( + order2, + currency_pair_btcusdt, + Some(StrategyId::new("S-001").unwrap()), + None, + Some(PositionId::new("P-123456").unwrap()), + Some(Price::from("45500.00")), + None, + None, + None, + ); + let result2 = cash_account_multi + .calculate_pnls(currency_pair_btcusdt, fill2, Some(position)) + .unwrap(); + // use hash set to ignore order of results + let result1_set: HashSet = result1.into_iter().collect(); + let result1_expected: HashSet = + vec![Money::from("22750 USDT"), Money::from("-0.5 BTC")] + .into_iter() + .collect(); + let result2_set: HashSet = result2.into_iter().collect(); + let result2_expected: HashSet = + vec![Money::from("-22750 USDT"), Money::from("0.5 BTC")] + .into_iter() + .collect(); + assert_eq!(result1_set, result1_expected); + assert_eq!(result2_set, result2_expected); + } + + #[rstest] + #[case(false, Money::from("-0.00218331 BTC"))] + #[case(true, Money::from("-25.0 USD"))] + fn test_calculate_commission_for_inverse_maker_crypto( + #[case] use_quote_for_inverse: bool, + #[case] expected: Money, + cash_account_million_usd: CashAccount, + xbtusd_bitmex: CryptoPerpetual, + ) { + let result = cash_account_million_usd + .calculate_commission( + xbtusd_bitmex, + Quantity::from("100000"), + Price::from("11450.50"), + LiquiditySide::Maker, + Some(use_quote_for_inverse), + ) + .unwrap(); + assert_eq!(result, expected); + } + + #[rstest] + fn test_calculate_commission_for_taker_fx( + cash_account_million_usd: CashAccount, + audusd_sim: CurrencyPair, + ) { + let result = cash_account_million_usd + .calculate_commission( + audusd_sim, + Quantity::from("1500000"), + Price::from("0.8005"), + LiquiditySide::Taker, + None, + ) + .unwrap(); + assert_eq!(result, Money::from("24.02 USD")); + } + + #[rstest] + fn test_calculate_commission_crypto_taker( + cash_account_million_usd: CashAccount, + xbtusd_bitmex: CryptoPerpetual, + ) { + let result = cash_account_million_usd + .calculate_commission( + xbtusd_bitmex, + Quantity::from("100000"), + Price::from("11450.50"), + LiquiditySide::Taker, + None, + ) + .unwrap(); + assert_eq!(result, Money::from("0.00654993 BTC")); + } + + #[rstest] + fn test_calculate_commission_fx_taker(cash_account_million_usd: CashAccount) { + let instrument = usdjpy_idealpro(); + let result = cash_account_million_usd + .calculate_commission( + instrument, + Quantity::from("2200000"), + Price::from("120.310"), + LiquiditySide::Taker, + None, + ) + .unwrap(); + assert_eq!(result, Money::from("5294 JPY")); + } +} diff --git a/nautilus_core/model/src/accounting/margin.rs b/nautilus_core/accounting/src/account/margin.rs similarity index 72% rename from nautilus_core/model/src/accounting/margin.rs rename to nautilus_core/accounting/src/account/margin.rs index 5ed625b4a788..811f32917a34 100644 --- a/nautilus_core/model/src/accounting/margin.rs +++ b/nautilus_core/accounting/src/account/margin.rs @@ -15,89 +15,52 @@ #![allow(dead_code)] +use std::ops::{Deref, DerefMut}; use std::{ collections::HashMap, fmt::Display, hash::{Hash, Hasher}, }; +use crate::account::base::BaseAccount; +use crate::account::Account; +use crate::position::Position; use anyhow::Result; +use nautilus_model::enums::{AccountType, LiquiditySide, OrderSide}; +use nautilus_model::events::account::state::AccountState; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::identifiers::instrument_id::InstrumentId; +use nautilus_model::instruments::Instrument; +use nautilus_model::types::balance::{AccountBalance, MarginBalance}; +use nautilus_model::types::currency::Currency; +use nautilus_model::types::money::Money; +use nautilus_model::types::price::Price; +use nautilus_model::types::quantity::Quantity; use pyo3::prelude::*; use rust_decimal::prelude::ToPrimitive; -use crate::{ - enums::AccountType, - events::account::state::AccountState, - identifiers::{account_id::AccountId, instrument_id::InstrumentId}, - instruments::Instrument, - types::{ - balance::{AccountBalance, MarginBalance}, - currency::Currency, - money::Money, - price::Price, - quantity::Quantity, - }, -}; - #[derive(Debug)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyclass(module = "nautilus_trader.core.nautilus_pyo3.accounting") )] pub struct MarginAccount { - pub id: AccountId, - pub account_type: AccountType, - pub base_currency: Currency, - calculate_account_state: bool, - events: Vec, - commissions: HashMap, - balances: HashMap, - balances_starting: HashMap, - margins: HashMap, + pub base: BaseAccount, pub leverages: HashMap, + pub margins: HashMap, pub default_leverage: f64, } impl MarginAccount { pub fn new(event: AccountState, calculate_account_state: bool) -> Result { - let mut balances_starting: HashMap = HashMap::new(); - let mut balances: HashMap = HashMap::new(); - event.balances.iter().for_each(|balance| { - balances_starting.insert(balance.currency, balance.total); - balances.insert(balance.currency, *balance); - }); - let mut margins: HashMap = HashMap::new(); - event.margins.iter().for_each(|margin| { - margins.insert(margin.instrument_id, *margin); - }); Ok(Self { - id: event.account_id, - account_type: event.account_type, - base_currency: event.base_currency, - calculate_account_state, - events: vec![event], - commissions: HashMap::new(), - balances, - balances_starting, - margins: HashMap::new(), + base: BaseAccount::new(event, calculate_account_state)?, leverages: HashMap::new(), + margins: HashMap::new(), default_leverage: 1.0, }) } - pub fn update_balances(&mut self, balances: Vec) { - balances.into_iter().for_each(|balance| { - // clone real balance without reference - if balance.total.raw < 0 { - // TODO raise AccountBalanceNegative event - panic!("Cannot update balances with total less than 0.0") - } else { - // clear asset balance - self.balances.insert(balance.currency, balance); - } - }); - } - pub fn set_default_leverage(&mut self, leverage: f64) { self.default_leverage = leverage; } @@ -283,6 +246,98 @@ impl MarginAccount { } } +impl Deref for MarginAccount { + type Target = BaseAccount; + + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl DerefMut for MarginAccount { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.base + } +} + +impl Account for MarginAccount { + fn balance_total(&self, currency: Option) -> Option { + self.base_balance_total(currency) + } + fn balances_total(&self) -> HashMap { + self.base_balances_total() + } + + fn balance_free(&self, currency: Option) -> Option { + self.base_balance_free(currency) + } + fn balances_free(&self) -> HashMap { + self.base_balances_free() + } + + fn balance_locked(&self, currency: Option) -> Option { + self.base_balance_locked(currency) + } + fn balances_locked(&self) -> HashMap { + self.base_balances_locked() + } + fn last_event(&self) -> Option { + self.base_last_event() + } + fn events(&self) -> Vec { + self.events.clone() + } + fn event_count(&self) -> usize { + self.events.len() + } + fn currencies(&self) -> Vec { + self.balances.keys().copied().collect() + } + fn starting_balances(&self) -> HashMap { + self.balances_starting.clone() + } + fn balances(&self) -> HashMap { + self.balances.clone() + } + fn apply(&mut self, event: AccountState) { + self.base_apply(event) + } + fn calculate_balance_locked( + &mut self, + instrument: T, + side: OrderSide, + quantity: Quantity, + price: Price, + use_quote_for_inverse: Option, + ) -> Result { + self.base_calculate_balance_locked(instrument, side, quantity, price, use_quote_for_inverse) + } + fn calculate_pnls( + &self, + instrument: T, + fill: OrderFilled, + position: Option, + ) -> Result> { + self.base_calculate_pnls(instrument, fill, position) + } + fn calculate_commission( + &self, + instrument: T, + last_qty: Quantity, + last_px: Price, + liquidity_side: LiquiditySide, + use_quote_for_inverse: Option, + ) -> Result { + self.base_calculate_commission( + instrument, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + } +} + impl PartialEq for MarginAccount { fn eq(&self, other: &Self) -> bool { self.id == other.id @@ -296,7 +351,11 @@ impl Display for MarginAccount { write!( f, "MarginAccount(id={}, type={}, base={})", - self.id, self.account_type, self.base_currency.code + self.id, + self.account_type, + self.base_currency + .map(|base_currency| format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), ) } } @@ -312,18 +371,22 @@ impl Hash for MarginAccount { //////////////////////////////////////////////////////////////////////////////// #[cfg(test)] mod tests { + use crate::account::margin::MarginAccount; + use crate::account::stubs::*; + use crate::account::Account; + use nautilus_model::events::account::state::AccountState; + use nautilus_model::events::account::stubs::*; + use nautilus_model::identifiers::instrument_id::InstrumentId; + use nautilus_model::identifiers::stubs::*; + use nautilus_model::instruments::crypto_perpetual::CryptoPerpetual; + use nautilus_model::instruments::currency_pair::CurrencyPair; + use nautilus_model::instruments::stubs::*; + use nautilus_model::types::currency::Currency; + use nautilus_model::types::money::Money; + use nautilus_model::types::price::Price; + use nautilus_model::types::quantity::Quantity; use rstest::rstest; - - use crate::{ - accounting::{margin::MarginAccount, stubs::*}, - identifiers::{instrument_id::InstrumentId, stubs::*}, - instruments::{ - crypto_perpetual::CryptoPerpetual, - currency_pair::CurrencyPair, - stubs::{audusd_sim, xbtusd_bitmex}, - }, - types::{money::Money, price::Price, quantity::Quantity}, - }; + use std::collections::HashMap; #[rstest] fn test_display(margin_account: MarginAccount) { @@ -333,6 +396,41 @@ mod tests { ); } + #[rstest] + fn test_base_account_properties( + margin_account: MarginAccount, + margin_account_state: AccountState, + ) { + assert_eq!(margin_account.base_currency, Some(Currency::from("USD"))); + assert_eq!( + margin_account.last_event(), + Some(margin_account_state.clone()) + ); + assert_eq!(margin_account.events(), vec![margin_account_state.clone()]); + assert_eq!(margin_account.event_count(), 1); + assert_eq!( + margin_account.balance_total(None), + Some(Money::from("1525000 USD")) + ); + assert_eq!( + margin_account.balance_free(None), + Some(Money::from("1500000 USD")) + ); + assert_eq!( + margin_account.balance_locked(None), + Some(Money::from("25000 USD")) + ); + let mut balances_total_expected = HashMap::new(); + balances_total_expected.insert(Currency::from("USD"), Money::from("1525000 USD")); + assert_eq!(margin_account.balances_total(), balances_total_expected); + let mut balances_free_expected = HashMap::new(); + balances_free_expected.insert(Currency::from("USD"), Money::from("1500000 USD")); + assert_eq!(margin_account.balances_free(), balances_free_expected); + let mut balances_locked_expected = HashMap::new(); + balances_locked_expected.insert(Currency::from("USD"), Money::from("25000 USD")); + assert_eq!(margin_account.balances_locked(), balances_locked_expected); + } + #[rstest] fn test_set_default_leverage(mut margin_account: MarginAccount) { assert_eq!(margin_account.default_leverage, 1.0); diff --git a/nautilus_core/accounting/src/account/mod.rs b/nautilus_core/accounting/src/account/mod.rs new file mode 100644 index 000000000000..e5fd079e775c --- /dev/null +++ b/nautilus_core/accounting/src/account/mod.rs @@ -0,0 +1,75 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use crate::position::Position; +use anyhow::Result; +use nautilus_model::enums::{LiquiditySide, OrderSide}; +use nautilus_model::events::account::state::AccountState; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::instruments::Instrument; +use nautilus_model::types::balance::AccountBalance; +use nautilus_model::types::currency::Currency; +use nautilus_model::types::money::Money; +use nautilus_model::types::price::Price; +use nautilus_model::types::quantity::Quantity; +use std::collections::HashMap; + +pub trait Account { + fn balance_total(&self, currency: Option) -> Option; + fn balances_total(&self) -> HashMap; + fn balance_free(&self, currency: Option) -> Option; + fn balances_free(&self) -> HashMap; + + fn balance_locked(&self, currency: Option) -> Option; + fn balances_locked(&self) -> HashMap; + fn last_event(&self) -> Option; + fn events(&self) -> Vec; + fn event_count(&self) -> usize; + fn currencies(&self) -> Vec; + fn starting_balances(&self) -> HashMap; + fn balances(&self) -> HashMap; + fn apply(&mut self, event: AccountState); + fn calculate_balance_locked( + &mut self, + instrument: T, + side: OrderSide, + quantity: Quantity, + price: Price, + use_quote_for_inverse: Option, + ) -> Result; + + fn calculate_pnls( + &self, + instrument: T, + fill: OrderFilled, + position: Option, + ) -> Result>; + + fn calculate_commission( + &self, + instrument: T, + last_qty: Quantity, + last_px: Price, + liquidity_side: LiquiditySide, + use_quote_for_inverse: Option, + ) -> Result; +} + +pub mod base; +pub mod cash; +pub mod margin; + +#[cfg(test)] +pub mod stubs; diff --git a/nautilus_core/model/src/accounting/stubs.rs b/nautilus_core/accounting/src/account/stubs.rs similarity index 61% rename from nautilus_core/model/src/accounting/stubs.rs rename to nautilus_core/accounting/src/account/stubs.rs index a27192575371..10d67cc4dada 100644 --- a/nautilus_core/model/src/accounting/stubs.rs +++ b/nautilus_core/accounting/src/account/stubs.rs @@ -13,14 +13,29 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +use nautilus_model::events::account::state::AccountState; use rstest::fixture; -use crate::{ - accounting::margin::MarginAccount, - events::account::{state::AccountState, stubs::*}, -}; +use crate::account::cash::CashAccount; +use crate::account::margin::MarginAccount; +use nautilus_model::events::account::stubs::*; #[fixture] pub fn margin_account(margin_account_state: AccountState) -> MarginAccount { MarginAccount::new(margin_account_state, true).unwrap() } + +#[fixture] +pub fn cash_account(cash_account_state: AccountState) -> CashAccount { + CashAccount::new(cash_account_state, true).unwrap() +} + +#[fixture] +pub fn cash_account_million_usd(cash_account_state_million_usd: AccountState) -> CashAccount { + CashAccount::new(cash_account_state_million_usd, true).unwrap() +} + +#[fixture] +pub fn cash_account_multi(cash_account_state_multi: AccountState) -> CashAccount { + CashAccount::new(cash_account_state_multi, true).unwrap() +} diff --git a/nautilus_core/model/src/python/accounting/mod.rs b/nautilus_core/accounting/src/lib.rs similarity index 91% rename from nautilus_core/model/src/python/accounting/mod.rs rename to nautilus_core/accounting/src/lib.rs index d0c09c95d89c..e8e6a930e7ea 100644 --- a/nautilus_core/model/src/python/accounting/mod.rs +++ b/nautilus_core/accounting/src/lib.rs @@ -13,4 +13,8 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -pub mod margin; +pub mod account; +pub mod position; + +#[cfg(feature = "python")] +pub mod python; diff --git a/nautilus_core/accounting/src/position.rs b/nautilus_core/accounting/src/position.rs new file mode 100644 index 000000000000..fed955dd62cb --- /dev/null +++ b/nautilus_core/accounting/src/position.rs @@ -0,0 +1,126 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use anyhow::Result; +use nautilus_core::time::UnixNanos; +use nautilus_model::enums::{OrderSide, PositionSide}; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::identifiers::account_id::AccountId; +use nautilus_model::identifiers::client_order_id::ClientOrderId; +use nautilus_model::identifiers::instrument_id::InstrumentId; +use nautilus_model::identifiers::position_id::PositionId; +use nautilus_model::identifiers::strategy_id::StrategyId; +use nautilus_model::identifiers::trade_id::TradeId; +use nautilus_model::identifiers::trader_id::TraderId; +use nautilus_model::identifiers::venue_order_id::VenueOrderId; +use nautilus_model::instruments::Instrument; +use nautilus_model::types::currency::Currency; +use nautilus_model::types::money::Money; +use nautilus_model::types::quantity::Quantity; +use pyo3::prelude::*; +use std::collections::HashMap; + +/// Represents a position in a financial market. +/// +/// The position ID may be assigned at the trading venue, or can be system +/// generated depending on a strategies OMS (Order Management System) settings. +#[repr(C)] +#[derive(Debug, Clone)] +#[cfg_attr( + feature = "python", + pyclass(module = "nautilus_trader.core.nautilus_pyo3.common") +)] +pub struct Position { + events: Vec, + pub trader_id: TraderId, + pub strategy_id: StrategyId, + pub instrument_id: InstrumentId, + pub id: PositionId, + pub account_id: AccountId, + pub opening_order_id: ClientOrderId, + pub closing_order_id: Option, + pub entry: OrderSide, + pub side: PositionSide, + pub signed_qty: f64, + pub quantity: Quantity, + pub peak_qty: Quantity, + pub price_precision: u8, + pub size_precision: u8, + pub multiplier: Quantity, + pub is_inverse: bool, + pub base_currency: Option, + pub quote_currency: Currency, + pub settlement_currency: Currency, + pub ts_init: UnixNanos, + pub ts_opened: UnixNanos, + pub ts_last: UnixNanos, + pub ts_closed: Option, + pub duration_ns: u64, + pub avg_px_open: f64, + pub avg_px_close: Option, + pub realized_return: Option, + pub realized_pnl: Option, + venue_order_ids: Vec, + trade_ids: Vec, + buy_qty: Quantity, + sell_qty: Quantity, + commissions: HashMap, +} + +impl Position { + pub fn new(instrument: T, fill: OrderFilled) -> Result { + assert_eq!(instrument.id(), &fill.instrument_id); + assert!(fill.position_id.is_some()); + assert_ne!(fill.order_side, OrderSide::NoOrderSide); + + let item = Self { + events: Vec::::new(), + venue_order_ids: Vec::::new(), + trade_ids: Vec::::new(), + buy_qty: Quantity::zero(instrument.size_precision()), + sell_qty: Quantity::zero(instrument.size_precision()), + commissions: HashMap::::new(), + trader_id: fill.trader_id, + strategy_id: fill.strategy_id, + instrument_id: fill.instrument_id, + id: fill.position_id.unwrap(), // TODO: Improve validation + account_id: fill.account_id, + opening_order_id: fill.client_order_id, + closing_order_id: None, + entry: fill.order_side, + side: PositionSide::Flat, + signed_qty: 0.0, + quantity: fill.last_qty, + peak_qty: fill.last_qty, + price_precision: instrument.price_precision(), + size_precision: instrument.size_precision(), + multiplier: instrument.multiplier(), + is_inverse: instrument.is_inverse(), + base_currency: instrument.base_currency().to_owned().copied(), + quote_currency: *instrument.quote_currency(), + settlement_currency: *instrument.settlement_currency(), + ts_init: fill.ts_init, + ts_opened: fill.ts_event, + ts_last: fill.ts_event, + ts_closed: None, + duration_ns: 0, + avg_px_open: fill.last_px.as_f64(), + avg_px_close: None, + realized_return: None, + realized_pnl: None, + }; + Ok(item) + } +} diff --git a/nautilus_core/accounting/src/python/cash.rs b/nautilus_core/accounting/src/python/cash.rs new file mode 100644 index 000000000000..20625f2d3163 --- /dev/null +++ b/nautilus_core/accounting/src/python/cash.rs @@ -0,0 +1,361 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use crate::account::cash::CashAccount; +use crate::account::Account; +use crate::position::Position; +use nautilus_core::python::to_pyvalue_err; +use nautilus_model::enums::{LiquiditySide, OrderSide}; +use nautilus_model::events::account::state::AccountState; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::identifiers::account_id::AccountId; +use nautilus_model::instruments::crypto_future::CryptoFuture; +use nautilus_model::instruments::crypto_perpetual::CryptoPerpetual; +use nautilus_model::instruments::currency_pair::CurrencyPair; +use nautilus_model::instruments::equity::Equity; +use nautilus_model::instruments::futures_contract::FuturesContract; +use nautilus_model::instruments::options_contract::OptionsContract; +use nautilus_model::types::currency::Currency; +use nautilus_model::types::money::Money; +use nautilus_model::types::price::Price; +use nautilus_model::types::quantity::Quantity; +use pyo3::basic::CompareOp; +use pyo3::prelude::*; +use std::collections::HashMap; + +#[pymethods] +impl CashAccount { + #[new] + pub fn py_new(event: AccountState, calculate_account_state: bool) -> PyResult { + Self::new(event, calculate_account_state).map_err(to_pyvalue_err) + } + + fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + match op { + CompareOp::Eq => self.eq(other).into_py(py), + CompareOp::Ne => self.ne(other).into_py(py), + _ => py.NotImplemented(), + } + } + + #[getter] + fn id(&self) -> AccountId { + self.id + } + + fn __str__(&self) -> String { + format!( + "{}(id={}, type={}, base={})", + stringify!(CashAccount), + self.id, + self.account_type, + self.base_currency + .map(|base_currency| format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), + ) + } + + fn __repr__(&self) -> String { + format!( + "{}(id={}, type={}, base={})", + stringify!(CashAccount), + self.id, + self.account_type, + self.base_currency + .map(|base_currency| format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), + ) + } + + #[getter] + fn base_currency(&self) -> Option { + self.base_currency + } + + #[getter] + #[pyo3(name = "last_event")] + fn py_last_event(&self) -> Option { + self.last_event() + } + + #[getter] + #[pyo3(name = "event_count")] + fn py_event_count(&self) -> usize { + self.event_count() + } + + #[getter] + #[pyo3(name = "events")] + fn py_events(&self) -> Vec { + self.events() + } + + #[pyo3(name = "balance_total")] + fn py_balance_total(&self, currency: Option) -> Option { + self.balance_total(currency) + } + + #[pyo3(name = "balances_total")] + fn py_balances_total(&self) -> HashMap { + self.balances_total() + } + + #[pyo3(name = "balance_free")] + fn py_balance_free(&self, currency: Option) -> Option { + self.balance_free(currency) + } + + #[pyo3(name = "balances_free")] + fn py_balances_free(&self) -> HashMap { + self.balances_free() + } + + #[pyo3(name = "balance_locked")] + fn py_balance_locked(&self, currency: Option) -> Option { + self.balance_locked(currency) + } + #[pyo3(name = "balances_locked")] + fn py_balances_locked(&self) -> HashMap { + self.balances_locked() + } + + #[pyo3(name = "apply")] + fn py_apply(&mut self, event: AccountState) { + self.apply(event) + } + + #[pyo3(name = "calculate_balance_locked")] + fn py_calculate_balance_locked( + &mut self, + instrument: PyObject, + side: OrderSide, + quantity: Quantity, + price: Price, + use_quote_for_inverse: Option, + py: Python, + ) -> PyResult { + // extract instrument from PyObject + let instrument_type = instrument + .getattr(py, "instrument_type")? + .extract::(py)?; + if instrument_type == "CryptoFuture" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_balance_locked( + instrument_rust, + side, + quantity, + price, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "CryptoPerpetual" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_balance_locked( + instrument_rust, + side, + quantity, + price, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "CurrencyPair" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_balance_locked( + instrument_rust, + side, + quantity, + price, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "Equity" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_balance_locked( + instrument_rust, + side, + quantity, + price, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "FuturesContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_balance_locked( + instrument_rust, + side, + quantity, + price, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "OptionsContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_balance_locked( + instrument_rust, + side, + quantity, + price, + use_quote_for_inverse, + ) + .unwrap()) + } else { + // throw error unsupported instrument + Err(to_pyvalue_err("Unsupported instrument type")) + } + } + + #[pyo3(name = "calculate_commission")] + fn py_calculate_commission( + &self, + instrument: PyObject, + last_qty: Quantity, + last_px: Price, + liquidity_side: LiquiditySide, + use_quote_for_inverse: Option, + py: Python, + ) -> PyResult { + if liquidity_side == LiquiditySide::NoLiquiditySide { + return Err(to_pyvalue_err("Invalid liquidity side")); + } + // extract instrument from PyObject + let instrument_type = instrument + .getattr(py, "instrument_type")? + .extract::(py)?; + if instrument_type == "CryptoFuture" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_commission( + instrument_rust, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "CurrencyPair" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_commission( + instrument_rust, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "CryptoPerpetual" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_commission( + instrument_rust, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "Equity" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_commission( + instrument_rust, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "FuturesContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_commission( + instrument_rust, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + .unwrap()) + } else if instrument_type == "OptionsContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_commission( + instrument_rust, + last_qty, + last_px, + liquidity_side, + use_quote_for_inverse, + ) + .unwrap()) + } else { + // throw error unsupported instrument + Err(to_pyvalue_err("Unsupported instrument type")) + } + } + + #[pyo3(name = "calculate_pnls")] + fn py_calculate_pnls( + &self, + instrument: PyObject, + fill: OrderFilled, + position: Option, + py: Python, + ) -> PyResult> { + // extract instrument from PyObject + let instrument_type = instrument + .getattr(py, "instrument_type")? + .extract::(py)?; + if instrument_type == "CryptoFuture" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_pnls(instrument_rust, fill, position) + .unwrap()) + } else if instrument_type == "CurrencyPair" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_pnls(instrument_rust, fill, position) + .unwrap()) + } else if instrument_type == "CryptoPerpetual" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_pnls(instrument_rust, fill, position) + .unwrap()) + } else if instrument_type == "Equity" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_pnls(instrument_rust, fill, position) + .unwrap()) + } else if instrument_type == "FuturesContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_pnls(instrument_rust, fill, position) + .unwrap()) + } else if instrument_type == "OptionsContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(self + .calculate_pnls(instrument_rust, fill, position) + .unwrap()) + } else { + // throw error unsupported instrument + Err(to_pyvalue_err("Unsupported instrument type")) + } + } +} diff --git a/nautilus_core/model/src/python/accounting/margin.rs b/nautilus_core/accounting/src/python/margin.rs similarity index 90% rename from nautilus_core/model/src/python/accounting/margin.rs rename to nautilus_core/accounting/src/python/margin.rs index eaaf4ddc8940..73659cae5b68 100644 --- a/nautilus_core/model/src/python/accounting/margin.rs +++ b/nautilus_core/accounting/src/python/margin.rs @@ -13,21 +13,22 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +use crate::account::margin::MarginAccount; use nautilus_core::python::to_pyvalue_err; +use nautilus_model::events::account::state::AccountState; +use nautilus_model::identifiers::account_id::AccountId; +use nautilus_model::identifiers::instrument_id::InstrumentId; +use nautilus_model::instruments::crypto_future::CryptoFuture; +use nautilus_model::instruments::crypto_perpetual::CryptoPerpetual; +use nautilus_model::instruments::currency_pair::CurrencyPair; +use nautilus_model::instruments::equity::Equity; +use nautilus_model::instruments::futures_contract::FuturesContract; +use nautilus_model::instruments::options_contract::OptionsContract; +use nautilus_model::types::money::Money; +use nautilus_model::types::price::Price; +use nautilus_model::types::quantity::Quantity; use pyo3::{basic::CompareOp, prelude::*, types::PyDict}; -use crate::{ - accounting::margin::MarginAccount, - events::account::state::AccountState, - identifiers::{account_id::AccountId, instrument_id::InstrumentId}, - instruments::{ - crypto_future::CryptoFuture, crypto_perpetual::CryptoPerpetual, - currency_pair::CurrencyPair, equity::Equity, futures_contract::FuturesContract, - options_contract::OptionsContract, - }, - types::{money::Money, price::Price, quantity::Quantity}, -}; - #[pymethods] impl MarginAccount { #[new] @@ -59,7 +60,9 @@ impl MarginAccount { stringify!(MarginAccount), self.id, self.account_type, - self.base_currency.code + self.base_currency + .map(|base_currency| format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()) ) } @@ -69,7 +72,9 @@ impl MarginAccount { stringify!(MarginAccount), self.id, self.account_type, - self.base_currency.code + self.base_currency + .map(|base_currency| format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), ) } diff --git a/nautilus_core/model/src/accounting/mod.rs b/nautilus_core/accounting/src/python/mod.rs similarity index 73% rename from nautilus_core/model/src/accounting/mod.rs rename to nautilus_core/accounting/src/python/mod.rs index 88f1f6c55a64..1d93578bd60d 100644 --- a/nautilus_core/model/src/accounting/mod.rs +++ b/nautilus_core/accounting/src/python/mod.rs @@ -13,6 +13,16 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +use pyo3::{prelude::*, pymodule}; + +pub mod cash; pub mod margin; -#[cfg(feature = "stubs")] -pub mod stubs; +pub mod position; + +#[pymodule] +pub fn accounting(_: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/nautilus_core/accounting/src/python/position.rs b/nautilus_core/accounting/src/python/position.rs new file mode 100644 index 000000000000..ac41065126c2 --- /dev/null +++ b/nautilus_core/accounting/src/python/position.rs @@ -0,0 +1,54 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use crate::position::Position; +use nautilus_core::python::to_pyvalue_err; +use nautilus_model::events::order::filled::OrderFilled; +use nautilus_model::instruments::crypto_future::CryptoFuture; +use nautilus_model::instruments::currency_pair::CurrencyPair; +use nautilus_model::instruments::equity::Equity; +use nautilus_model::instruments::futures_contract::FuturesContract; +use nautilus_model::instruments::options_contract::OptionsContract; +use pyo3::prelude::*; + +#[pymethods] +impl Position { + #[new] + fn py_new(instrument: PyObject, fill: OrderFilled, py: Python) -> PyResult { + // extract instrument from PyObject + let instrument_type = instrument + .getattr(py, "instrument_type")? + .extract::(py)?; + if instrument_type == "CryptoFuture" { + let instrument_rust = instrument.extract::(py)?; + Ok(Self::new(instrument_rust, fill).unwrap()) + } else if instrument_type == "CurrencyPair" { + let instrument_rust = instrument.extract::(py)?; + Ok(Self::new(instrument_rust, fill).unwrap()) + } else if instrument_type == "Equity" { + let instrument_rust = instrument.extract::(py)?; + Ok(Self::new(instrument_rust, fill).unwrap()) + } else if instrument_type == "FuturesContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(Self::new(instrument_rust, fill).unwrap()) + } else if instrument_type == "OptionsContract" { + let instrument_rust = instrument.extract::(py)?; + Ok(Self::new(instrument_rust, fill).unwrap()) + } else { + // throw error unsupported instrument + Err(to_pyvalue_err("Unsupported instrument type")) + } + } +} diff --git a/nautilus_core/common/Cargo.toml b/nautilus_core/common/Cargo.toml index 56635a837493..6b5efda76aae 100644 --- a/nautilus_core/common/Cargo.toml +++ b/nautilus_core/common/Cargo.toml @@ -23,6 +23,7 @@ serde = { workspace = true } serde_json = { workspace = true } strum = { workspace = true } ustr = { workspace = true } +rstest = { workspace = true , optional = true} tracing = { workspace = true } sysinfo = "0.30.5" # Disable default feature "tracing-log" since it interferes with custom logging @@ -30,7 +31,6 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features = [dev-dependencies] tempfile = { workspace = true } -rstest = { workspace = true } [features] extension-module = [ @@ -40,6 +40,7 @@ extension-module = [ ] ffi = ["cbindgen"] python = ["pyo3"] +stubs = ["rstest"] redis = ["dep:redis"] default = ["ffi", "python", "redis"] diff --git a/nautilus_core/common/src/lib.rs b/nautilus_core/common/src/lib.rs index ff294294a8ad..cd5ae5b67f2f 100644 --- a/nautilus_core/common/src/lib.rs +++ b/nautilus_core/common/src/lib.rs @@ -24,7 +24,7 @@ pub mod msgbus; pub mod testing; pub mod timer; -#[cfg(test)] +#[cfg(feature = "stubs")] pub mod stubs; #[cfg(feature = "ffi")] diff --git a/nautilus_core/model/src/enums.rs b/nautilus_core/model/src/enums.rs index 055f7540eb09..6ffafc2fc416 100644 --- a/nautilus_core/model/src/enums.rs +++ b/nautilus_core/model/src/enums.rs @@ -542,6 +542,7 @@ pub enum InstrumentCloseType { #[allow(clippy::enum_variant_names)] pub enum LiquiditySide { /// No specific liqudity side. + #[pyo3(name = "NO_LIQUIDITY_SIDE")] NoLiquiditySide = 0, // Will be replaced by `Option` /// The order passively provided liqudity to the market to complete the trade (made a market). #[pyo3(name = "MAKER")] diff --git a/nautilus_core/model/src/events/account/state.rs b/nautilus_core/model/src/events/account/state.rs index 3369f340869d..cfdf2703fa2a 100644 --- a/nautilus_core/model/src/events/account/state.rs +++ b/nautilus_core/model/src/events/account/state.rs @@ -38,7 +38,7 @@ use crate::{ pub struct AccountState { pub account_id: AccountId, pub account_type: AccountType, - pub base_currency: Currency, + pub base_currency: Option, pub balances: Vec, pub margins: Vec, pub is_reported: bool, @@ -52,13 +52,13 @@ impl AccountState { pub fn new( account_id: AccountId, account_type: AccountType, - base_currency: Currency, balances: Vec, margins: Vec, is_reported: bool, event_id: UUID4, ts_event: UnixNanos, ts_init: UnixNanos, + base_currency: Option, ) -> Result { Ok(AccountState { account_id, @@ -81,7 +81,9 @@ impl Display for AccountState { "AccountState(account_id={}, account_type={}, base_currency={}, is_reported={}, balances=[{}], margins=[{}], event_id={})", self.account_id, self.account_type, - self.base_currency.code, + self.base_currency + .map(|base_currency | format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), self.is_reported, self.balances.iter().map(|b| format!("{}", b)).collect::>().join(","), self.margins.iter().map(|m| format!("{}", m)).collect::>().join(","), diff --git a/nautilus_core/model/src/events/account/stubs.rs b/nautilus_core/model/src/events/account/stubs.rs index 987d4cb10f27..8679d8f146f2 100644 --- a/nautilus_core/model/src/events/account/stubs.rs +++ b/nautilus_core/model/src/events/account/stubs.rs @@ -15,6 +15,8 @@ use rstest::fixture; +use crate::types::balance::AccountBalance; +use crate::types::money::Money; use crate::{ enums::AccountType, events::account::state::AccountState, @@ -30,13 +32,90 @@ pub fn cash_account_state() -> AccountState { AccountState::new( account_id(), AccountType::Cash, - Currency::USD(), vec![account_balance_test()], vec![], true, uuid4(), 0, 0, + Some(Currency::USD()), + ) + .unwrap() +} + +#[fixture] +pub fn cash_account_state_million_usd() -> AccountState { + AccountState::new( + account_id(), + AccountType::Cash, + vec![AccountBalance::new( + Money::from("1000000 USD"), + Money::from("0 USD"), + Money::from("1000000 USD"), + ) + .unwrap()], + vec![], + true, + uuid4(), + 0, + 0, + Some(Currency::USD()), + ) + .unwrap() +} + +#[fixture] +pub fn cash_account_state_multi() -> AccountState { + let btc_account_balance = AccountBalance::new( + Money::from("10 BTC"), + Money::from("0 BTC"), + Money::from("10 BTC"), + ) + .unwrap(); + let eth_account_balance = AccountBalance::new( + Money::from("20 ETH"), + Money::from("0 ETH"), + Money::from("20 ETH"), + ) + .unwrap(); + AccountState::new( + account_id(), + AccountType::Cash, + vec![btc_account_balance, eth_account_balance], + vec![], + true, + uuid4(), + 0, + 0, + None, // multi cash account + ) + .unwrap() +} + +#[fixture] +pub fn cash_account_state_multi_changed_btc() -> AccountState { + let btc_account_balance = AccountBalance::new( + Money::from("9 BTC"), + Money::from("0.5 BTC"), + Money::from("8.5 BTC"), + ) + .unwrap(); + let eth_account_balance = AccountBalance::new( + Money::from("20 ETH"), + Money::from("0 ETH"), + Money::from("20 ETH"), + ) + .unwrap(); + AccountState::new( + account_id(), + AccountType::Cash, + vec![btc_account_balance, eth_account_balance], + vec![], + true, + uuid4(), + 0, + 0, + None, // multi cash account ) .unwrap() } @@ -46,13 +125,13 @@ pub fn margin_account_state() -> AccountState { AccountState::new( account_id(), AccountType::Margin, - Currency::USD(), vec![account_balance_test()], vec![margin_balance_test()], true, uuid4(), 0, 0, + Some(Currency::USD()), ) .unwrap() } diff --git a/nautilus_core/model/src/instruments/stubs.rs b/nautilus_core/model/src/instruments/stubs.rs index 487b4c9e9f9d..483c8c90e89d 100644 --- a/nautilus_core/model/src/instruments/stubs.rs +++ b/nautilus_core/model/src/instruments/stubs.rs @@ -192,6 +192,11 @@ pub fn audusd_sim() -> CurrencyPair { default_fx_ccy(Symbol::from("AUD/USD"), Some(Venue::from("SIM"))) } +#[fixture] +pub fn usdjpy_idealpro() -> CurrencyPair { + default_fx_ccy(Symbol::from("USD/JPY"), Some(Venue::from("IDEALPRO"))) +} + //////////////////////////////////////////////////////////////////////////////// // Equity //////////////////////////////////////////////////////////////////////////////// diff --git a/nautilus_core/model/src/lib.rs b/nautilus_core/model/src/lib.rs index bfe332de3be1..6114ad7b6fb6 100644 --- a/nautilus_core/model/src/lib.rs +++ b/nautilus_core/model/src/lib.rs @@ -13,7 +13,6 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -pub mod accounting; pub mod currencies; pub mod data; pub mod enums; diff --git a/nautilus_core/model/src/orders/stubs.rs b/nautilus_core/model/src/orders/stubs.rs index d23b4da551a1..a942c15009f4 100644 --- a/nautilus_core/model/src/orders/stubs.rs +++ b/nautilus_core/model/src/orders/stubs.rs @@ -35,8 +35,8 @@ pub struct TestOrderEventStubs; impl TestOrderEventStubs { #[allow(clippy::too_many_arguments)] pub fn order_filled( - order: &T, - instrument: &I, + order: T, + instrument: I, strategy_id: Option, trade_id: Option, position_id: Option, diff --git a/nautilus_core/model/src/python/events/account/state.rs b/nautilus_core/model/src/python/events/account/state.rs index cbff84870535..c468f98c2e33 100644 --- a/nautilus_core/model/src/python/events/account/state.rs +++ b/nautilus_core/model/src/python/events/account/state.rs @@ -38,24 +38,24 @@ impl AccountState { fn py_new( account_id: AccountId, account_type: AccountType, - base_currency: Currency, balances: Vec, margins: Vec, is_reported: bool, event_id: UUID4, ts_event: UnixNanos, ts_init: UnixNanos, + base_currency: Option, ) -> PyResult { AccountState::new( account_id, account_type, - base_currency, balances, margins, is_reported, event_id, ts_event, ts_init, + base_currency, ) .map_err(to_pyvalue_err) } @@ -74,8 +74,9 @@ impl AccountState { stringify!(AccountState), self.account_id, self.account_type, - self.base_currency.code, - self.balances.iter().map(|b| format!("{}", b)).collect::>().join(","), + self.base_currency + .map(|base_currency | format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), self.balances.iter().map(|b| format!("{}", b)).collect::>().join(","), self.margins.iter().map(|m| format!("{}", m)).collect::>().join(","), self.is_reported, self.event_id, @@ -88,8 +89,9 @@ impl AccountState { stringify!(AccountState), self.account_id, self.account_type, - self.base_currency.code, - self.balances.iter().map(|b| format!("{}", b)).collect::>().join(","), + self.base_currency + .map(|base_currency | format!("{}", base_currency.code)) + .unwrap_or_else(|| "None".to_string()), self.balances.iter().map(|b| format!("{}", b)).collect::>().join(","), self.margins.iter().map(|m| format!("{}", m)).collect::>().join(","), self.is_reported, self.event_id, @@ -108,7 +110,6 @@ impl AccountState { dict.set_item("type", stringify!(AccountState))?; dict.set_item("account_id", self.account_id.to_string())?; dict.set_item("account_type", self.account_type.to_string())?; - dict.set_item("base_currency", self.base_currency.code.to_string())?; // iterate over balances and margins and run to_dict on each item and collect them let balances_dict: PyResult> = self.balances.iter().map(|b| b.py_to_dict(py)).collect(); @@ -120,6 +121,12 @@ impl AccountState { dict.set_item("event_id", self.event_id.to_string())?; dict.set_item("ts_event", self.ts_event.to_u64())?; dict.set_item("ts_init", self.ts_init.to_u64())?; + match self.base_currency { + Some(base_currency) => { + dict.set_item("base_currency", base_currency.code.to_string())? + } + None => dict.set_item("base_currency", "None")?, + } Ok(dict.into()) } } diff --git a/nautilus_core/model/src/python/mod.rs b/nautilus_core/model/src/python/mod.rs index a763526b8c33..63b68c151572 100644 --- a/nautilus_core/model/src/python/mod.rs +++ b/nautilus_core/model/src/python/mod.rs @@ -32,8 +32,6 @@ pub mod macros; pub mod orders; pub mod types; -pub mod accounting; - pub const PY_MODULE_MODEL: &str = "nautilus_trader.core.nautilus_pyo3.model"; /// Python iterator over the variants of an enum. @@ -323,7 +321,5 @@ pub fn model(_: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; // Events - account m.add_class::()?; - // Accounting - m.add_class::()?; Ok(()) } diff --git a/nautilus_core/model/src/python/orders/market.rs b/nautilus_core/model/src/python/orders/market.rs index 8c9e596cf525..54274dceef7d 100644 --- a/nautilus_core/model/src/python/orders/market.rs +++ b/nautilus_core/model/src/python/orders/market.rs @@ -20,6 +20,8 @@ use pyo3::prelude::*; use rust_decimal::Decimal; use ustr::Ustr; +use crate::enums::OrderType; +use crate::identifiers::account_id::AccountId; use crate::{ enums::{ContingencyType, OrderSide, PositionSide, TimeInForce}, identifiers::{ @@ -135,4 +137,24 @@ impl MarketOrder { fn py_commissions(&self) -> HashMap { self.commissions() } + #[getter] + fn account_id(&self) -> Option { + self.account_id + } + #[getter] + fn client_order_id(&self) -> ClientOrderId { + self.client_order_id + } + #[getter] + fn quantity(&self) -> Quantity { + self.quantity + } + #[getter] + fn side(&self) -> OrderSide { + self.side + } + #[getter] + fn order_type(&self) -> OrderType { + self.order_type + } } diff --git a/nautilus_core/pyo3/Cargo.toml b/nautilus_core/pyo3/Cargo.toml index 8b95d70be569..358818e9a8d6 100644 --- a/nautilus_core/pyo3/Cargo.toml +++ b/nautilus_core/pyo3/Cargo.toml @@ -19,6 +19,7 @@ nautilus-infrastructure = { path = "../infrastructure" } nautilus-model = { path = "../model" } nautilus-persistence = { path = "../persistence" } nautilus-network = { path = "../network" } +nautilus-accounting = { path = "../accounting" } pyo3 = { workspace = true } [features] diff --git a/nautilus_core/pyo3/src/lib.rs b/nautilus_core/pyo3/src/lib.rs index ea28254a9125..de97adfa42d0 100644 --- a/nautilus_core/pyo3/src/lib.rs +++ b/nautilus_core/pyo3/src/lib.rs @@ -104,6 +104,12 @@ fn nautilus_pyo3(py: Python<'_>, m: &PyModule) -> PyResult<()> { sys_modules.set_item(format!("{module_name}.{n}"), m.getattr(n)?)?; re_export_module_attributes(m, n)?; + let n = "accounting"; + let submodule = pyo3::wrap_pymodule!(nautilus_accounting::python::accounting); + m.add_wrapped(submodule)?; + sys_modules.set_item(format!("{module_name}.{n}"), m.getattr(n)?)?; + re_export_module_attributes(m, n)?; + Ok(()) } diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 0f06f0a8f593..868bc0db262d 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -7,7 +7,7 @@ from collections.abc import Callable from decimal import Decimal from enum import Enum from os import PathLike -from typing import Any, TypeAlias +from typing import Any, TypeAlias, Union from nautilus_trader.core.data import Data @@ -216,6 +216,13 @@ def init_logging( ################################################################################################### ### Accounting + +class Position: + def __init__( + self, + instrument: CurrencyPair | CryptoPerpetual | Equity | OptionsContract | SyntheticInstrument, + fill: OrderFilled + ) -> None: ... class MarginAccount: def __init__( self, @@ -256,6 +263,55 @@ class MarginAccount: use_quote_for_inverse: bool | None = None ) -> Money: ... +class CashAccount: + def __init__( + self, + event: AccountState, + calculate_account_state: bool + ) -> None: ... + + @property + def id(self) -> AccountId: ... + @property + def base_currency(self) -> Currency | None: ... + @property + def last_event(self) -> AccountState | None: ... + def events(self) -> list[AccountState]: ... + @property + def event_count(self) -> int: ... + def balance_total(self, currency: Currency | None) -> Money | None : ... + def balances_total(self) -> dict[Currency,Money]: ... + def balance_free(self, currency: Currency | None) -> Money | None : ... + + def balances_free(self) -> dict[Currency,Money]: ... + def balance_locked(self, currency: Currency | None) -> Money | None: ... + + def balances_locked(self) -> dict[Currency,Money]: ... + def apply(self, event: AccountState): ... + def calculate_balance_locked( + self, + instrument: Instrument, + side: OrderSide, + quantity: Quantity, + price: Price, + use_quote_for_inverse: bool | None = None + ) -> Money: ... + def calculate_commission( + self, + instrument: Instrument, + last_qty: Quantity, + last_px: Price, + liquidity_side: LiquiditySide, + use_quote_for_inverse: bool | None = None + ) -> Money: ... + def calculate_pnls( + self, + instrument: Instrument, + fill: OrderFilled, + position: Position | None = None + ) -> list[Money]: ... + + ### Data types class BarSpecification: @@ -494,6 +550,7 @@ class InstrumentCloseType(Enum): class LiquiditySide(Enum): MAKER = "MAKER" TAKER = "TAKER" + NO_LIQUIDITY_SIDE = "NO_LIQUIDITY_SIDE" class MarketStatus(Enum): PRE_OPEN = "PRE_OPEN" @@ -604,6 +661,7 @@ class ClientId: class ClientOrderId: def __init__(self, value: str) -> None: ... + @property def value(self) -> str: ... class ComponentId: @@ -693,6 +751,30 @@ class MarketOrder: def would_reduce_only(self, side: PositionSide, position_qty: Quantity) -> bool: ... def commission(self, currency: Currency) -> Money | None: ... def commissions(self) -> dict[Currency, Money]: ... + @property + def trader_id(self) -> TraderId: ... + @property + def account_id(self) -> AccountId: ... + @property + def strategy_id(self) -> StrategyId: ... + @property + def instrument_id(self) -> InstrumentId: ... + @property + def client_order_id(self) -> ClientOrderId: ... + @property + def venue_order_id(self) -> VenueOrderId | None: ... + @property + def position_id(self) -> PositionId | None: ... + @property + def last_trade_id(self) -> TradeId | None: ... + @property + def quantity(self) -> Quantity: ... + @property + def side(self) -> OrderSide: ... + @property + def order_type(self) -> OrderType: ... + @property + def price(self) -> Price | None: ... class MarketToLimitOrder: ... class StopLimitOrder: ... @@ -847,6 +929,20 @@ class CryptoFuture: @property def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... class CryptoPerpetual: def __init__( @@ -878,6 +974,20 @@ class CryptoPerpetual: @property def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... class CurrencyPair: def __init__( @@ -905,6 +1015,20 @@ class CurrencyPair: @property def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... class Equity: def __init__( @@ -926,6 +1050,20 @@ class Equity: @property def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... class FuturesContract: def __init__( @@ -951,6 +1089,20 @@ class FuturesContract: @property def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... class OptionsContract: def __init__( @@ -978,12 +1130,49 @@ class OptionsContract: @property def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... class SyntheticInstrument: def id(self) -> InstrumentId: ... def to_dict(self) -> dict[str, Any]: ... + @property + def symbol(self) -> Symbol: ... + @property + def price_precision(self) -> int: ... + @property + def size_precision(self) -> int: ... + @property + def price_increment(self) -> Price: ... + @property + def size_increment(self) -> Quantity: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... + +Instrument: TypeAlias = Union[ + CryptoFuture, + CryptoPerpetual, + CurrencyPair, + Equity, + FuturesContract, + OptionsContract, + SyntheticInstrument, +] -Instrument: TypeAlias = Equity | FuturesContract | OptionsContract | CryptoFuture | CryptoPerpetual | CurrencyPair | SyntheticInstrument ### Events diff --git a/nautilus_trader/test_kit/rust/accounting_pyo3.py b/nautilus_trader/test_kit/rust/accounting_pyo3.py index 7f43f711e5ac..53b35ab948fc 100644 --- a/nautilus_trader/test_kit/rust/accounting_pyo3.py +++ b/nautilus_trader/test_kit/rust/accounting_pyo3.py @@ -13,6 +13,7 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- +from nautilus_trader.core.nautilus_pyo3 import CashAccount from nautilus_trader.core.nautilus_pyo3 import MarginAccount from nautilus_trader.test_kit.rust.events_pyo3 import TestEventsProviderPyo3 @@ -24,3 +25,24 @@ def margin_account() -> MarginAccount: event=TestEventsProviderPyo3.margin_account_state(), calculate_account_state=False, ) + + @staticmethod + def cash_account() -> CashAccount: + return CashAccount( + event=TestEventsProviderPyo3.cash_account_state(), + calculate_account_state=False, + ) + + @staticmethod + def cash_account_million_usd() -> CashAccount: + return CashAccount( + event=TestEventsProviderPyo3.cash_account_state_million_usd(), + calculate_account_state=False, + ) + + @staticmethod + def cash_account_multi() -> CashAccount: + return CashAccount( + event=TestEventsProviderPyo3.cash_account_state_multi(), + calculate_account_state=False, + ) diff --git a/nautilus_trader/test_kit/rust/events_pyo3.py b/nautilus_trader/test_kit/rust/events_pyo3.py index 94777329e28a..ff3a6c6f0fa5 100644 --- a/nautilus_trader/test_kit/rust/events_pyo3.py +++ b/nautilus_trader/test_kit/rust/events_pyo3.py @@ -14,12 +14,19 @@ # ------------------------------------------------------------------------------------------------- from nautilus_trader.core.nautilus_pyo3 import UUID4 +from nautilus_trader.core.nautilus_pyo3 import AccountBalance +from nautilus_trader.core.nautilus_pyo3 import AccountId from nautilus_trader.core.nautilus_pyo3 import AccountState from nautilus_trader.core.nautilus_pyo3 import AccountType +from nautilus_trader.core.nautilus_pyo3 import CashAccount from nautilus_trader.core.nautilus_pyo3 import ClientOrderId from nautilus_trader.core.nautilus_pyo3 import ContingencyType +from nautilus_trader.core.nautilus_pyo3 import CryptoFuture +from nautilus_trader.core.nautilus_pyo3 import CryptoPerpetual from nautilus_trader.core.nautilus_pyo3 import Currency +from nautilus_trader.core.nautilus_pyo3 import CurrencyPair from nautilus_trader.core.nautilus_pyo3 import LiquiditySide +from nautilus_trader.core.nautilus_pyo3 import MarketOrder from nautilus_trader.core.nautilus_pyo3 import Money from nautilus_trader.core.nautilus_pyo3 import OrderAccepted from nautilus_trader.core.nautilus_pyo3 import OrderCanceled @@ -43,9 +50,11 @@ from nautilus_trader.core.nautilus_pyo3 import PositionId from nautilus_trader.core.nautilus_pyo3 import Price from nautilus_trader.core.nautilus_pyo3 import Quantity +from nautilus_trader.core.nautilus_pyo3 import StrategyId from nautilus_trader.core.nautilus_pyo3 import TimeInForce from nautilus_trader.core.nautilus_pyo3 import TradeId from nautilus_trader.core.nautilus_pyo3 import TriggerType +from nautilus_trader.core.nautilus_pyo3 import VenueOrderId from nautilus_trader.test_kit.rust.identifiers_pyo3 import TestIdProviderPyo3 from nautilus_trader.test_kit.rust.types_pyo3 import TestTypesProviderPyo3 @@ -70,6 +79,76 @@ def cash_account_state() -> AccountState: ts_event=0, ) + @staticmethod + def cash_account_state_million_usd() -> AccountState: + return AccountState( + account_id=TestIdProviderPyo3.account_id(), + account_type=AccountType.CASH, + base_currency=Currency.from_str("USD"), + balances=[ + TestTypesProviderPyo3.account_balance( + total=Money.from_str("1000000 USD"), + locked=Money.from_str("0 USD"), + free=Money.from_str("1000000 USD"), + ), + ], + margins=[], + is_reported=True, + event_id=UUID4("91762096-b188-49ea-8562-8d8a4cc22ff2"), + ts_init=0, + ts_event=0, + ) + + @staticmethod + def cash_account_state_multi() -> AccountState: + return AccountState( + account_id=TestIdProviderPyo3.account_id(), + account_type=AccountType.CASH, + base_currency=None, + balances=[ + AccountBalance( + total=Money.from_str("10 BTC"), + locked=Money.from_str("0 BTC"), + free=Money.from_str("10 BTC"), + ), + AccountBalance( + total=Money.from_str("20 ETH"), + locked=Money.from_str("0 ETH"), + free=Money.from_str("20 ETH"), + ), + ], + margins=[], + is_reported=True, + event_id=UUID4("91762096-b188-49ea-8562-8d8a4cc22ff2"), + ts_init=0, + ts_event=0, + ) + + @staticmethod + def cash_account_state_multi_changed_btc() -> AccountState: + return AccountState( + account_id=TestIdProviderPyo3.account_id(), + account_type=AccountType.CASH, + base_currency=None, + balances=[ + AccountBalance( + total=Money.from_str("9 BTC"), + locked=Money.from_str("0.5 BTC"), + free=Money.from_str("8.5 BTC"), + ), + AccountBalance( + total=Money.from_str("20 ETH"), + locked=Money.from_str("0 ETH"), + free=Money.from_str("20 ETH"), + ), + ], + margins=[], + is_reported=True, + event_id=UUID4("91762096-b188-49ea-8562-8d8a4cc22ff2"), + ts_init=0, + ts_event=0, + ) + @staticmethod def margin_account_state() -> AccountState: return AccountState( @@ -348,3 +427,67 @@ def order_expired() -> OrderExpired: ts_event=0, reconciliation=False, ) + + @staticmethod + def order_filled( + order: MarketOrder, + instrument: CurrencyPair | CryptoPerpetual | CryptoFuture, + strategy_id: StrategyId | None = None, + account_id: AccountId | None = None, + venue_order_id: VenueOrderId | None = None, + trade_id: TradeId | None = None, + position_id: PositionId | None = None, + last_qty: Quantity | None = None, + last_px: Price | None = None, + liquidity_side: LiquiditySide = LiquiditySide.TAKER, + ts_filled_ns: int = 0, + account: CashAccount | None = None, + ) -> OrderFilled: + if strategy_id is None: + strategy_id = order.strategy_id + if account_id is None: + account_id = order.account_id + if account_id is None: + account_id = TestIdProviderPyo3.account_id() + if venue_order_id is None: + venue_order_id = VenueOrderId("1") + if trade_id is None: + trade_id = TradeId(order.client_order_id.value.replace("O", "E")) + if position_id is None: + position_id = order.position_id + if last_px is None: + last_px = Price.from_str(f"{1:.{instrument.price_precision}f}") + if last_qty is None: + last_qty = order.quantity + if account is None: + from nautilus_trader.test_kit.rust.accounting_pyo3 import TestAccountingProviderPyo3 + + account = TestAccountingProviderPyo3.cash_account() + assert account is not None + commission = account.calculate_commission( + instrument=instrument, + last_qty=order.quantity, + last_px=last_px, + liquidity_side=liquidity_side, + ) + return OrderFilled( + trader_id=TestIdProviderPyo3.trader_id(), + strategy_id=strategy_id, + instrument_id=instrument.id, + client_order_id=order.client_order_id, + venue_order_id=venue_order_id, + account_id=account_id, + trade_id=trade_id, + position_id=position_id, + order_side=order.side, + order_type=order.order_type, + last_qty=last_qty, + last_px=last_px or order.price, + currency=instrument.quote_currency, + commission=commission, + liquidity_side=liquidity_side, + ts_event=ts_filled_ns, + event_id=TestIdProviderPyo3.uuid(), + ts_init=0, + reconciliation=False, + ) diff --git a/tests/unit_tests/accounting/test_cash_pyo3.py b/tests/unit_tests/accounting/test_cash_pyo3.py new file mode 100644 index 000000000000..7e786f7ff09a --- /dev/null +++ b/tests/unit_tests/accounting/test_cash_pyo3.py @@ -0,0 +1,244 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- +import pytest + +from nautilus_trader.core.nautilus_pyo3 import AccountId +from nautilus_trader.core.nautilus_pyo3 import Currency +from nautilus_trader.core.nautilus_pyo3 import LiquiditySide +from nautilus_trader.core.nautilus_pyo3 import Money +from nautilus_trader.core.nautilus_pyo3 import OrderSide +from nautilus_trader.core.nautilus_pyo3 import Position +from nautilus_trader.core.nautilus_pyo3 import Price +from nautilus_trader.core.nautilus_pyo3 import Quantity +from nautilus_trader.test_kit.rust.accounting_pyo3 import TestAccountingProviderPyo3 +from nautilus_trader.test_kit.rust.events_pyo3 import TestEventsProviderPyo3 +from nautilus_trader.test_kit.rust.identifiers_pyo3 import TestIdProviderPyo3 +from nautilus_trader.test_kit.rust.instruments_pyo3 import TestInstrumentProviderPyo3 +from nautilus_trader.test_kit.rust.orders_pyo3 import TestOrderProviderPyo3 + + +AUDUSD_SIM = TestIdProviderPyo3.audusd_id() +AUD_USD = TestInstrumentProviderPyo3.default_fx_ccy("AUD/USD") +USD_JPY = TestInstrumentProviderPyo3.default_fx_ccy("USD/JPY") +USD = Currency.from_str("USD") +BTC = Currency.from_str("BTC") +ETH = Currency.from_str("ETH") +AUD = Currency.from_str("AUD") +JPY = Currency.from_str("JPY") + + +def test_instantiated_account_basic_properties(): + account = TestAccountingProviderPyo3.cash_account() + + assert account.id == AccountId("SIM-000") + assert str(account) == "CashAccount(id=SIM-000, type=CASH, base=USD)" + assert repr(account) == "CashAccount(id=SIM-000, type=CASH, base=USD)" + assert account == account + + +def test_instantiate_single_asset_cash_account(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + event = TestEventsProviderPyo3.cash_account_state_million_usd() + + assert account.base_currency == Currency.from_str("USD") + assert account.last_event == TestEventsProviderPyo3.cash_account_state_million_usd() + assert account.events == [event] + assert account.event_count == 1 + assert account.balance_total() == Money(1_000_000, USD) + assert account.balance_free() == Money(1_000_000, USD) + assert account.balance_locked() == Money(0, USD) + assert account.balances_total() == {USD: Money(1_000_000, USD)} + assert account.balances_free() == {USD: Money(1_000_000, USD)} + assert account.balances_locked() == {USD: Money(0, USD)} + + +def test_instantiate_multi_asset_cash_account(): + account = TestAccountingProviderPyo3.cash_account_multi() + event = TestEventsProviderPyo3.cash_account_state_multi() + + assert account.id == AccountId("SIM-000") + assert account.base_currency is None + assert account.last_event == event + assert account.event_count == 1 + assert account.events == [event] + assert account.balance_total(BTC) == Money(10.00000000, BTC) + assert account.balance_total(ETH) == Money(20.00000000, ETH) + assert account.balance_free(BTC) == Money(10.00000000, BTC) + assert account.balance_free(ETH) == Money(20.00000000, ETH) + assert account.balance_locked(BTC) == Money(0.00000000, BTC) + assert account.balance_locked(ETH) == Money(0.00000000, ETH) + assert account.balances_total() == { + BTC: Money(10.00000000, BTC), + ETH: Money(20.00000000, ETH), + } + assert account.balances_free() == { + BTC: Money(10.00000000, BTC), + ETH: Money(20.00000000, ETH), + } + assert account.balances_locked() == { + BTC: Money(0.00000000, BTC), + ETH: Money(0.00000000, ETH), + } + + +def test_apply_given_new_state_event_updates_correctly(): + account = TestAccountingProviderPyo3.cash_account_multi() + event = TestEventsProviderPyo3.cash_account_state_multi() + new_event = TestEventsProviderPyo3.cash_account_state_multi_changed_btc() + + account.apply(new_event) + + assert account.last_event == new_event + assert account.events == [event, new_event] + assert account.event_count == 2 + assert account.balance_total(BTC) == Money(9.00000000, BTC) + assert account.balance_free(BTC) == Money(8.50000000, BTC) + assert account.balance_locked(BTC) == Money(0.50000000, BTC) + assert account.balance_total(ETH) == Money(20.00000000, ETH) + assert account.balance_free(ETH) == Money(20.00000000, ETH) + assert account.balance_locked(ETH) == Money(0.00000000, ETH) + + +def test_calculate_balance_locked_buy(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + result = account.calculate_balance_locked( + instrument=AUD_USD, + side=OrderSide.BUY, + quantity=Quantity.from_int(1_000_000), + price=Price.from_str("0.80"), + ) + assert result == Money(800_032.00, USD) # Notional + expected commission + + +def test_calculate_balance_locked_sell(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + result = account.calculate_balance_locked( + instrument=AUD_USD, + side=OrderSide.SELL, + quantity=Quantity.from_int(1_000_000), + price=Price.from_str("0.80"), + ) + assert result == Money(1_000_040.00, AUD) # Notional + expected commission + + +def test_calculate_balance_locked_sell_no_base_currency(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + result = account.calculate_balance_locked( + instrument=TestInstrumentProviderPyo3.aapl_equity(), + side=OrderSide.SELL, + quantity=Quantity.from_int(100), + price=Price.from_str("1500.00"), + ) + assert result == Money(100.00, USD) # Notional + expected commission + + +def test_calculate_pnls_for_single_currency_cash_account(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + order = TestOrderProviderPyo3.market_order( + instrument_id=AUD_USD.id, + order_side=OrderSide.BUY, + quantity=Quantity.from_int(1_000_000), + ) + fill = TestEventsProviderPyo3.order_filled( + order=order, + instrument=AUD_USD, + position_id=TestIdProviderPyo3.position_id(), + strategy_id=TestIdProviderPyo3.strategy_id(), + last_px=Price.from_str("0.80"), + ) + position = Position(AUD_USD, fill) + result = account.calculate_pnls( + instrument=AUD_USD, + fill=fill, + position=position, + ) + assert result == [Money(-800000.00, USD)] + + +def test_calculate_commission_when_given_liquidity_side_none_raises_value_error(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + instrument = TestInstrumentProviderPyo3.xbtusd_bitmex() + with pytest.raises(ValueError): + account.calculate_commission( + instrument=instrument, + last_qty=Quantity.from_int(100_000), + last_px=Price.from_str("11450.50"), + liquidity_side=LiquiditySide.NO_LIQUIDITY_SIDE, + ) + + +@pytest.mark.parametrize( + ("use_quote_for_inverse", "expected"), + [ + [False, Money(-0.00218331, BTC)], # Negative commission = credit + [True, Money(-25.00, USD)], # Negative commission = credit + ], +) +def test_calculate_commission_for_inverse_maker_crypto(use_quote_for_inverse, expected): + account = TestAccountingProviderPyo3.cash_account_million_usd() + instrument = TestInstrumentProviderPyo3.xbtusd_bitmex() + + result = account.calculate_commission( + instrument=instrument, + last_qty=Quantity.from_int(100_000), + last_px=Price.from_str("11450.50"), + liquidity_side=LiquiditySide.MAKER, + use_quote_for_inverse=use_quote_for_inverse, + ) + + assert result == expected + + +def test_calculate_commission_for_taker_fx(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + instrument = AUD_USD + + result = account.calculate_commission( + instrument=instrument, + last_qty=Quantity.from_int(1_500_000), + last_px=Price.from_str("0.80050"), + liquidity_side=LiquiditySide.TAKER, + ) + + assert result == Money(24.02, USD) + + +def test_calculate_commission_crypto_taker(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + instrument = TestInstrumentProviderPyo3.xbtusd_bitmex() + + result = account.calculate_commission( + instrument=instrument, + last_qty=Quantity.from_int(100_000), + last_px=Price.from_str("11450.50"), + liquidity_side=LiquiditySide.TAKER, + ) + + assert result == Money(0.00654993, BTC) + + +def test_calculate_commission_fx_taker(): + account = TestAccountingProviderPyo3.cash_account_million_usd() + + # Act + result = account.calculate_commission( + instrument=USD_JPY, + last_qty=Quantity.from_int(2_200_000), + last_px=Price.from_str("120.310"), + liquidity_side=LiquiditySide.TAKER, + ) + + # Assert + assert result == Money(5294, JPY) diff --git a/tests/unit_tests/accounting/test_margin_pyo3.py b/tests/unit_tests/accounting/test_margin_pyo3.py index d1814bdabd55..72cd7a79a049 100644 --- a/tests/unit_tests/accounting/test_margin_pyo3.py +++ b/tests/unit_tests/accounting/test_margin_pyo3.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- + import pytest from nautilus_trader.core.nautilus_pyo3 import AccountId @@ -29,7 +30,7 @@ BTC = Currency.from_str("BTC") -def test_instantiated_accounts_basic_properties(): +def test_instantiated_account_basic_properties(): account = TestAccountingProviderPyo3.margin_account() assert account.id == AccountId("SIM-000") From 84a0680eada353a3447877e7507d82aa025e7fec Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Thu, 1 Feb 2024 15:34:53 +1100 Subject: [PATCH 39/42] Update dependencies --- nautilus_core/Cargo.lock | 72 ++++++++++++++++---------------- nautilus_core/Cargo.toml | 8 ++-- nautilus_core/network/Cargo.toml | 2 +- poetry.lock | 25 +++++------ 4 files changed, 54 insertions(+), 53 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 92be58006640..acac21061bd4 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -266,7 +266,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.2.1", + "indexmap 2.2.2", "lexical-core", "num", "serde", @@ -1150,7 +1150,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.3", - "indexmap 2.2.1", + "indexmap 2.2.2", "itertools 0.12.1", "log", "num_cpus", @@ -1266,7 +1266,7 @@ dependencies = [ "half", "hashbrown 0.14.3", "hex", - "indexmap 2.2.1", + "indexmap 2.2.2", "itertools 0.12.1", "log", "md-5", @@ -1299,7 +1299,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.3", - "indexmap 2.2.1", + "indexmap 2.2.2", "itertools 0.12.1", "log", "once_cell", @@ -1765,7 +1765,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.1", + "indexmap 2.2.2", "slab", "tokio", "tokio-util", @@ -1784,7 +1784,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.1", + "indexmap 2.2.2", "slab", "tokio", "tokio-util", @@ -2019,12 +2019,11 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", - "futures-channel", "futures-util", "http 1.0.0", "http-body 1.0.0", @@ -2032,7 +2031,6 @@ dependencies = [ "pin-project-lite", "socket2 0.5.5", "tokio", - "tracing", ] [[package]] @@ -2092,9 +2090,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" +checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2256,9 +2254,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libm" @@ -2436,7 +2434,7 @@ dependencies = [ "criterion", "databento", "dbn", - "indexmap 2.2.1", + "indexmap 2.2.2", "itoa", "log", "nautilus-common", @@ -2479,7 +2477,7 @@ dependencies = [ "anyhow", "cbindgen", "chrono", - "indexmap 2.2.1", + "indexmap 2.2.2", "log", "nautilus-core", "nautilus-model", @@ -2554,7 +2552,7 @@ dependencies = [ "evalexpr", "float-cmp", "iai", - "indexmap 2.2.1", + "indexmap 2.2.2", "nautilus-core", "once_cell", "pyo3", @@ -2882,9 +2880,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.1+3.2.0" +version = "300.2.2+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" +checksum = "8bbfad0063610ac26ee79f7484739e2b07555a75c42453b89263830b5c8103bc" dependencies = [ "cc", ] @@ -3029,7 +3027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.1", + "indexmap 2.2.2", ] [[package]] @@ -3530,9 +3528,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64", "bytes", @@ -3552,9 +3550,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -3699,9 +3699,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.33.1" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +checksum = "d7de2711cae7bdec993f4d2319352599ceb0d003e9f7900ea7c6ef4c5fc16831" dependencies = [ "arrayvec", "borsh", @@ -3715,9 +3715,9 @@ dependencies = [ [[package]] name = "rust_decimal_macros" -version = "1.33.1" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e43721f4ef7060ebc2c3ede757733209564ca8207f47674181bcd425dd76945" +checksum = "69deb21b04afa2c06038f75bbbb5670a320e62ee005d91a00cf13fbf20161886" dependencies = [ "quote", "rust_decimal", @@ -4198,7 +4198,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.2.1", + "indexmap 2.2.2", "log", "memchr", "once_cell", @@ -4790,11 +4790,11 @@ checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.2", "toml_datetime", "winnow", ] @@ -5085,9 +5085,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cdbaf5e132e593e9fc1de6a15bbec912395b11fb9719e061cf64f804524c503" +checksum = "126e423afe2dd9ac52142e7e9d5ce4135d7e13776c529d27fd6bc49f19e3280b" [[package]] name = "vcpkg" @@ -5194,9 +5194,9 @@ checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -5414,9 +5414,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.35" +version = "0.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" +checksum = "818ce546a11a9986bc24f93d0cdf38a8a1a400f1473ea8c82e59f6e0ffab9249" dependencies = [ "memchr", ] diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 4d33d3ce348d..3a46ebbc5ba8 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "accounting", "adapters", "backtest", "common", @@ -12,7 +13,6 @@ members = [ "network/tokio-tungstenite", "persistence", "pyo3", - "accounting" ] [workspace.package] @@ -27,7 +27,7 @@ documentation = "https://docs.nautilustrader.io" anyhow = "1.0.79" chrono = "0.4.33" futures = "0.3.30" -indexmap = "2.2.1" +indexmap = "2.2.2" itoa = "1.0.10" once_cell = "1.19.0" log = { version = "0.4.20", features = ["std", "kv_unstable", "serde", "release_max_level_debug"] } @@ -36,8 +36,8 @@ pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime", "tokio", "attr rand = "0.8.5" redis = { version = "0.24.0", features = ["tokio-comp", "tls-rustls", "tokio-rustls-comp", "keep-alive", "connection-manager"] } rmp-serde = "1.1.2" -rust_decimal = "1.33.1" -rust_decimal_macros = "1.33.1" +rust_decimal = "1.34.0" +rust_decimal_macros = "1.34.0" serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.112" strum = { version = "0.25.0", features = ["derive"] } diff --git a/nautilus_core/network/Cargo.toml b/nautilus_core/network/Cargo.toml index b52c0202df79..d9b3e062c93e 100644 --- a/nautilus_core/network/Cargo.toml +++ b/nautilus_core/network/Cargo.toml @@ -23,7 +23,7 @@ futures-util = "0.3.29" http = "1.0.0" hyper = "1.1.0" nonzero_ext = "0.3.0" -reqwest = "0.11.23" +reqwest = "0.11.24" tokio-tungstenite = { path = "./tokio-tungstenite", features = ["rustls-tls-native-roots"] } [dev-dependencies] diff --git a/poetry.lock b/poetry.lock index b1eadc735d82..96b0ce7ff1a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -464,7 +464,7 @@ name = "css-html-js-minify" version = "2.5.5" description = "CSS HTML JS Minifier" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ {file = "css-html-js-minify-2.5.5.zip", hash = "sha256:4a9f11f7e0496f5284d12111f3ba4ff5ff2023d12f15d195c9c48bd97013746c"}, {file = "css_html_js_minify-2.5.5-py2.py3-none-any.whl", hash = "sha256:3da9d35ac0db8ca648c1b543e0e801d7ca0bab9e6bfd8418fee59d5ae001727a"}, @@ -1492,18 +1492,18 @@ files = [ [[package]] name = "platformdirs" -version = "4.1.0" +version = "4.2.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, - {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] [[package]] name = "pluggy" @@ -1791,13 +1791,13 @@ six = ">=1.5" [[package]] name = "python-slugify" -version = "8.0.2" +version = "8.0.3" description = "A Python slugify application that also handles Unicode" optional = false python-versions = ">=3.7" files = [ - {file = "python-slugify-8.0.2.tar.gz", hash = "sha256:a1a02b127a95c124fd84f8f88be730e557fd823774bf19b1cd5e8704e2ae0e5e"}, - {file = "python_slugify-8.0.2-py2.py3-none-any.whl", hash = "sha256:428ea9b00c977b8f6c097724398f190b2c18e2a6011094d1001285875ccacdbf"}, + {file = "python-slugify-8.0.3.tar.gz", hash = "sha256:e04cba5f1c562502a1175c84a8bc23890c54cdaf23fccaaf0bf78511508cabed"}, + {file = "python_slugify-8.0.3-py2.py3-none-any.whl", hash = "sha256:c71189c161e8c671f1b141034d9a56308a8a5978cd13d40446c879569212fdd1"}, ] [package.dependencies] @@ -2375,17 +2375,18 @@ files = [ [[package]] name = "urllib3" -version = "2.1.0" +version = "2.2.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, - {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, + {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, + {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, ] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] From 2429cf851b7c01621cb07a295358bc45e4b56963 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Thu, 1 Feb 2024 15:36:39 +1100 Subject: [PATCH 40/42] Update Databento publishers --- .../adapters/src/databento/publishers.json | 114 +++++++++++++++++- .../adapters/databento/publishers.json | 114 +++++++++++++++++- .../adapters/databento/test_loaders.py | 2 +- 3 files changed, 223 insertions(+), 7 deletions(-) diff --git a/nautilus_core/adapters/src/databento/publishers.json b/nautilus_core/adapters/src/databento/publishers.json index 5abdb766c350..f9f45ff0c837 100644 --- a/nautilus_core/adapters/src/databento/publishers.json +++ b/nautilus_core/adapters/src/databento/publishers.json @@ -9,19 +9,19 @@ "publisher_id": 2, "dataset": "XNAS.ITCH", "venue": "XNAS", - "description": "Nasdaq TotalView ITCH" + "description": "Nasdaq TotalView-ITCH" }, { "publisher_id": 3, "dataset": "XBOS.ITCH", "venue": "XBOS", - "description": "Nasdaq BX TotalView ITCH" + "description": "Nasdaq BX TotalView-ITCH" }, { "publisher_id": 4, "dataset": "XPSX.ITCH", "venue": "XPSX", - "description": "Nasdaq PSX TotalView ITCH" + "description": "Nasdaq PSX TotalView-ITCH" }, { "publisher_id": 5, @@ -256,5 +256,113 @@ "dataset": "ARCX.PILLAR", "venue": "ARCX", "description": "NYSE Arca Integrated" + }, + { + "publisher_id": 44, + "dataset": "XNYS.BBO", + "venue": "XNYS", + "description": "NYSE BBO" + }, + { + "publisher_id": 45, + "dataset": "XNYS.TRADES", + "venue": "XNYS", + "description": "NYSE Trades" + }, + { + "publisher_id": 46, + "dataset": "XNAS.QBBO", + "venue": "XNAS", + "description": "Nasdaq QBBO" + }, + { + "publisher_id": 47, + "dataset": "XNAS.NLS", + "venue": "XNAS", + "description": "Nasdaq Trades" + }, + { + "publisher_id": 48, + "dataset": "DBEQ.PLUS", + "venue": "XCHI", + "description": "DBEQ Plus - NYSE Chicago" + }, + { + "publisher_id": 49, + "dataset": "DBEQ.PLUS", + "venue": "XCIS", + "description": "DBEQ Plus - NYSE National" + }, + { + "publisher_id": 50, + "dataset": "DBEQ.PLUS", + "venue": "IEXG", + "description": "DBEQ Plus - IEX" + }, + { + "publisher_id": 51, + "dataset": "DBEQ.PLUS", + "venue": "EPRL", + "description": "DBEQ Plus - MIAX Pearl" + }, + { + "publisher_id": 52, + "dataset": "DBEQ.PLUS", + "venue": "XNAS", + "description": "DBEQ Plus - Nasdaq" + }, + { + "publisher_id": 53, + "dataset": "DBEQ.PLUS", + "venue": "XNYS", + "description": "DBEQ Plus - NYSE" + }, + { + "publisher_id": 54, + "dataset": "DBEQ.PLUS", + "venue": "FINN", + "description": "DBEQ Plus - FINRA/NYSE TRF" + }, + { + "publisher_id": 55, + "dataset": "DBEQ.PLUS", + "venue": "FINY", + "description": "DBEQ Plus - FINRA/Nasdaq TRF Carteret" + }, + { + "publisher_id": 56, + "dataset": "DBEQ.PLUS", + "venue": "FINC", + "description": "DBEQ Plus - FINRA/Nasdaq TRF Chicago" + }, + { + "publisher_id": 57, + "dataset": "IFEU.IMPACT", + "venue": "IFEU", + "description": "ICE Futures Europe (Commodities)" + }, + { + "publisher_id": 58, + "dataset": "NDEX.IMPACT", + "venue": "NDEX", + "description": "ICE Endex" + }, + { + "publisher_id": 59, + "dataset": "DBEQ.BASIC", + "venue": "DBEQ", + "description": "DBEQ Basic - Consolidated" + }, + { + "publisher_id": 60, + "dataset": "DBEQ.PLUS", + "venue": "DBEQ", + "description": "DBEQ Plus - Consolidated" + }, + { + "publisher_id": 61, + "dataset": "OPRA.PILLAR", + "venue": "SPHR", + "description": "OPRA - MIAX Sapphire" } ] diff --git a/nautilus_trader/adapters/databento/publishers.json b/nautilus_trader/adapters/databento/publishers.json index 5abdb766c350..f9f45ff0c837 100644 --- a/nautilus_trader/adapters/databento/publishers.json +++ b/nautilus_trader/adapters/databento/publishers.json @@ -9,19 +9,19 @@ "publisher_id": 2, "dataset": "XNAS.ITCH", "venue": "XNAS", - "description": "Nasdaq TotalView ITCH" + "description": "Nasdaq TotalView-ITCH" }, { "publisher_id": 3, "dataset": "XBOS.ITCH", "venue": "XBOS", - "description": "Nasdaq BX TotalView ITCH" + "description": "Nasdaq BX TotalView-ITCH" }, { "publisher_id": 4, "dataset": "XPSX.ITCH", "venue": "XPSX", - "description": "Nasdaq PSX TotalView ITCH" + "description": "Nasdaq PSX TotalView-ITCH" }, { "publisher_id": 5, @@ -256,5 +256,113 @@ "dataset": "ARCX.PILLAR", "venue": "ARCX", "description": "NYSE Arca Integrated" + }, + { + "publisher_id": 44, + "dataset": "XNYS.BBO", + "venue": "XNYS", + "description": "NYSE BBO" + }, + { + "publisher_id": 45, + "dataset": "XNYS.TRADES", + "venue": "XNYS", + "description": "NYSE Trades" + }, + { + "publisher_id": 46, + "dataset": "XNAS.QBBO", + "venue": "XNAS", + "description": "Nasdaq QBBO" + }, + { + "publisher_id": 47, + "dataset": "XNAS.NLS", + "venue": "XNAS", + "description": "Nasdaq Trades" + }, + { + "publisher_id": 48, + "dataset": "DBEQ.PLUS", + "venue": "XCHI", + "description": "DBEQ Plus - NYSE Chicago" + }, + { + "publisher_id": 49, + "dataset": "DBEQ.PLUS", + "venue": "XCIS", + "description": "DBEQ Plus - NYSE National" + }, + { + "publisher_id": 50, + "dataset": "DBEQ.PLUS", + "venue": "IEXG", + "description": "DBEQ Plus - IEX" + }, + { + "publisher_id": 51, + "dataset": "DBEQ.PLUS", + "venue": "EPRL", + "description": "DBEQ Plus - MIAX Pearl" + }, + { + "publisher_id": 52, + "dataset": "DBEQ.PLUS", + "venue": "XNAS", + "description": "DBEQ Plus - Nasdaq" + }, + { + "publisher_id": 53, + "dataset": "DBEQ.PLUS", + "venue": "XNYS", + "description": "DBEQ Plus - NYSE" + }, + { + "publisher_id": 54, + "dataset": "DBEQ.PLUS", + "venue": "FINN", + "description": "DBEQ Plus - FINRA/NYSE TRF" + }, + { + "publisher_id": 55, + "dataset": "DBEQ.PLUS", + "venue": "FINY", + "description": "DBEQ Plus - FINRA/Nasdaq TRF Carteret" + }, + { + "publisher_id": 56, + "dataset": "DBEQ.PLUS", + "venue": "FINC", + "description": "DBEQ Plus - FINRA/Nasdaq TRF Chicago" + }, + { + "publisher_id": 57, + "dataset": "IFEU.IMPACT", + "venue": "IFEU", + "description": "ICE Futures Europe (Commodities)" + }, + { + "publisher_id": 58, + "dataset": "NDEX.IMPACT", + "venue": "NDEX", + "description": "ICE Endex" + }, + { + "publisher_id": 59, + "dataset": "DBEQ.BASIC", + "venue": "DBEQ", + "description": "DBEQ Basic - Consolidated" + }, + { + "publisher_id": 60, + "dataset": "DBEQ.PLUS", + "venue": "DBEQ", + "description": "DBEQ Plus - Consolidated" + }, + { + "publisher_id": 61, + "dataset": "OPRA.PILLAR", + "venue": "SPHR", + "description": "OPRA - MIAX Sapphire" } ] diff --git a/tests/integration_tests/adapters/databento/test_loaders.py b/tests/integration_tests/adapters/databento/test_loaders.py index e454b77bdc24..0ff69fd54950 100644 --- a/tests/integration_tests/adapters/databento/test_loaders.py +++ b/tests/integration_tests/adapters/databento/test_loaders.py @@ -40,7 +40,7 @@ def test_get_publishers() -> None: result = loader.publishers # Assert - assert len(result) == 43 # From built-in map + assert len(result) == 61 # From built-in map # def test_loader_definition_glbx_futures() -> None: From 474cc6281675941c33abc3625d250f0e01c27fbb Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 2 Feb 2024 21:24:33 +1100 Subject: [PATCH 41/42] Update dependencies including black --- .pre-commit-config.yaml | 4 +- nautilus_core/Cargo.lock | 28 ++- nautilus_core/Cargo.toml | 4 +- nautilus_core/pyo3/src/lib.rs | 12 +- .../adapters/betfair/data_types.py | 12 +- .../adapters/betfair/parsing/requests.py | 32 ++-- nautilus_trader/adapters/betfair/providers.py | 8 +- .../adapters/binance/common/execution.py | 6 +- .../binance/common/schemas/account.py | 8 +- .../adapters/binance/common/types.py | 6 +- .../binance/futures/schemas/account.py | 4 +- .../binance/futures/schemas/market.py | 4 +- .../adapters/bybit/common/enums.py | 24 ++- .../interactive_brokers/client/client.py | 8 +- nautilus_trader/analysis/statistic.py | 2 +- nautilus_trader/backtest/node.py | 6 +- nautilus_trader/core/rust/algorithms.pxd | 5 +- nautilus_trader/core/rust/model.pyx | 5 +- nautilus_trader/data/client.pyx | 1 + .../persistence/catalog/parquet.py | 3 +- nautilus_trader/system/kernel.py | 8 +- poetry.lock | 174 ++++++++++-------- .../betfair/test_betfair_execution.py | 6 +- .../adapters/betfair/test_kit.py | 42 +++-- .../integration_tests/live/test_live_node.py | 12 +- 25 files changed, 243 insertions(+), 181 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7b26014e491..e10bebc10196 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: types: [python] - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.1.1 hooks: - id: black types_or: [python, pyi] @@ -82,7 +82,7 @@ repos: exclude: "docs/_pygments/monokai.py" - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.14 + rev: v0.1.15 hooks: - id: ruff args: ["--fix"] diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index acac21061bd4..54f8ea421ca2 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2730,6 +2730,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -3699,9 +3705,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.34.0" +version = "1.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7de2711cae7bdec993f4d2319352599ceb0d003e9f7900ea7c6ef4c5fc16831" +checksum = "755392e1a2f77afd95580d3f0d0e94ac83eeeb7167552c9b5bca549e61a94d83" dependencies = [ "arrayvec", "borsh", @@ -3715,9 +3721,9 @@ dependencies = [ [[package]] name = "rust_decimal_macros" -version = "1.34.0" +version = "1.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69deb21b04afa2c06038f75bbbb5670a320e62ee005d91a00cf13fbf20161886" +checksum = "e418701588729bef95e7a655f2b483ad64bb97c46e8e79fde83efd92aaab6d82" dependencies = [ "quote", "rust_decimal", @@ -3740,9 +3746,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -4606,12 +4612,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "fe80ced77cbfb4cb91a94bf72b378b4b6791a0d9b7f09d0be747d1bdff4e68bd" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -4626,10 +4633,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 3a46ebbc5ba8..c7b73433442b 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -36,8 +36,8 @@ pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime", "tokio", "attr rand = "0.8.5" redis = { version = "0.24.0", features = ["tokio-comp", "tls-rustls", "tokio-rustls-comp", "keep-alive", "connection-manager"] } rmp-serde = "1.1.2" -rust_decimal = "1.34.0" -rust_decimal_macros = "1.34.0" +rust_decimal = "1.34.2" +rust_decimal_macros = "1.34.2" serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.112" strum = { version = "0.25.0", features = ["derive"] } diff --git a/nautilus_core/pyo3/src/lib.rs b/nautilus_core/pyo3/src/lib.rs index de97adfa42d0..818b85f787ac 100644 --- a/nautilus_core/pyo3/src/lib.rs +++ b/nautilus_core/pyo3/src/lib.rs @@ -56,6 +56,12 @@ fn nautilus_pyo3(py: Python<'_>, m: &PyModule) -> PyResult<()> { // Set pyo3_nautilus to be recognized as a subpackage sys_modules.set_item(module_name, m)?; + let n = "accounting"; + let submodule = pyo3::wrap_pymodule!(nautilus_accounting::python::accounting); + m.add_wrapped(submodule)?; + sys_modules.set_item(format!("{module_name}.{n}"), m.getattr(n)?)?; + re_export_module_attributes(m, n)?; + let n = "databento"; let submodule = pyo3::wrap_pymodule!(databento); m.add_wrapped(submodule)?; @@ -104,12 +110,6 @@ fn nautilus_pyo3(py: Python<'_>, m: &PyModule) -> PyResult<()> { sys_modules.set_item(format!("{module_name}.{n}"), m.getattr(n)?)?; re_export_module_attributes(m, n)?; - let n = "accounting"; - let submodule = pyo3::wrap_pymodule!(nautilus_accounting::python::accounting); - m.add_wrapped(submodule)?; - sys_modules.set_item(format!("{module_name}.{n}"), m.getattr(n)?)?; - re_export_module_attributes(m, n)?; - Ok(()) } diff --git a/nautilus_trader/adapters/betfair/data_types.py b/nautilus_trader/adapters/betfair/data_types.py index 1b07048a2e9c..805dedabc7a7 100644 --- a/nautilus_trader/adapters/betfair/data_types.py +++ b/nautilus_trader/adapters/betfair/data_types.py @@ -196,12 +196,12 @@ def from_dict(cls, values: dict): ts_init=values["ts_init"], last_traded_price=values["last_traded_price"] if values["last_traded_price"] else None, traded_volume=values["traded_volume"] if values["traded_volume"] else None, - starting_price_near=values["starting_price_near"] - if values["starting_price_near"] - else None, - starting_price_far=values["starting_price_far"] - if values["starting_price_far"] - else None, + starting_price_near=( + values["starting_price_near"] if values["starting_price_near"] else None + ), + starting_price_far=( + values["starting_price_far"] if values["starting_price_far"] else None + ), ) @staticmethod diff --git a/nautilus_trader/adapters/betfair/parsing/requests.py b/nautilus_trader/adapters/betfair/parsing/requests.py index 067e91422e09..c8d2f5bf09d7 100644 --- a/nautilus_trader/adapters/betfair/parsing/requests.py +++ b/nautilus_trader/adapters/betfair/parsing/requests.py @@ -98,9 +98,11 @@ def nautilus_limit_to_place_instructions( instructions = PlaceInstruction( order_type=OrderType.LIMIT, selection_id=int(instrument.selection_id), - handicap=instrument.selection_handicap - if instrument.selection_handicap != null_handicap() - else None, + handicap=( + instrument.selection_handicap + if instrument.selection_handicap != null_handicap() + else None + ), side=OrderSideParser.to_betfair(command.order.side), limit_order=LimitOrder( price=command.order.price.as_double(), @@ -126,9 +128,11 @@ def nautilus_limit_on_close_to_place_instructions( instructions = PlaceInstruction( order_type=OrderType.LIMIT_ON_CLOSE, selection_id=int(instrument.selection_id), - handicap=instrument.selection_handicap - if instrument.selection_handicap != null_handicap() - else None, + handicap=( + instrument.selection_handicap + if instrument.selection_handicap != null_handicap() + else None + ), side=OrderSideParser.to_betfair(command.order.side), limit_on_close_order=LimitOnCloseOrder( price=command.order.price.as_double(), @@ -150,9 +154,11 @@ def nautilus_market_to_place_instructions( instructions = PlaceInstruction( order_type=OrderType.LIMIT, selection_id=int(instrument.selection_id), - handicap=instrument.selection_handicap - if instrument.selection_handicap != null_handicap() - else None, + handicap=( + instrument.selection_handicap + if instrument.selection_handicap != null_handicap() + else None + ), side=OrderSideParser.to_betfair(command.order.side), limit_order=LimitOrder( price=price.as_double(), @@ -178,9 +184,11 @@ def nautilus_market_on_close_to_place_instructions( instructions = PlaceInstruction( order_type=OrderType.MARKET_ON_CLOSE, selection_id=int(instrument.selection_id), - handicap=instrument.selection_handicap - if instrument.selection_handicap != null_handicap() - else None, + handicap=( + instrument.selection_handicap + if instrument.selection_handicap != null_handicap() + else None + ), side=OrderSideParser.to_betfair(command.order.side), market_on_close_order=MarketOnCloseOrder( liability=command.order.quantity.as_double(), diff --git a/nautilus_trader/adapters/betfair/providers.py b/nautilus_trader/adapters/betfair/providers.py index ff94091655c8..579b4bdf0a94 100644 --- a/nautilus_trader/adapters/betfair/providers.py +++ b/nautilus_trader/adapters/betfair/providers.py @@ -182,9 +182,11 @@ def market_definition_to_instruments( betting_type=market_definition.betting_type.name, market_id=market_definition.market_id, market_name=market_definition.market_name or "", - market_start_time=pd.Timestamp(market_definition.market_time) - if market_definition.market_time - else pd.Timestamp(0, tz="UTC"), + market_start_time=( + pd.Timestamp(market_definition.market_time) + if market_definition.market_time + else pd.Timestamp(0, tz="UTC") + ), market_type=market_definition.market_type, selection_id=runner.id, selection_name=runner.name or "", diff --git a/nautilus_trader/adapters/binance/common/execution.py b/nautilus_trader/adapters/binance/common/execution.py index 99b541057cf3..3f6596ff73a3 100644 --- a/nautilus_trader/adapters/binance/common/execution.py +++ b/nautilus_trader/adapters/binance/common/execution.py @@ -338,9 +338,9 @@ async def generate_order_status_report( else: binance_order = await self._http_account.query_order( symbol=instrument_id.symbol.value, - orig_client_order_id=client_order_id.value - if client_order_id is not None - else None, + orig_client_order_id=( + client_order_id.value if client_order_id is not None else None + ), ) except BinanceError as e: retries += 1 diff --git a/nautilus_trader/adapters/binance/common/schemas/account.py b/nautilus_trader/adapters/binance/common/schemas/account.py index 9e3c046e9ff4..6f867d539958 100644 --- a/nautilus_trader/adapters/binance/common/schemas/account.py +++ b/nautilus_trader/adapters/binance/common/schemas/account.py @@ -229,9 +229,11 @@ def parse_to_order_status_report( order_side=enum_parser.parse_binance_order_side(self.side), order_type=enum_parser.parse_binance_order_type(self.type), contingency_type=contingency_type, - time_in_force=enum_parser.parse_binance_time_in_force(self.timeInForce) - if self.timeInForce - else None, + time_in_force=( + enum_parser.parse_binance_time_in_force(self.timeInForce) + if self.timeInForce + else None + ), order_status=order_status, price=Price.from_str(self.price), trigger_price=Price.from_str(str(trigger_price)), # `decimal.Decimal` diff --git a/nautilus_trader/adapters/binance/common/types.py b/nautilus_trader/adapters/binance/common/types.py index 995ed3dc42a2..b1621d9a7406 100644 --- a/nautilus_trader/adapters/binance/common/types.py +++ b/nautilus_trader/adapters/binance/common/types.py @@ -429,9 +429,9 @@ def to_dict(obj: BinanceTicker) -> dict[str, Any]: "price_change": str(obj.price_change), "price_change_percent": str(obj.price_change_percent), "weighted_avg_price": str(obj.weighted_avg_price), - "prev_close_price": str(obj.prev_close_price) - if obj.prev_close_price is not None - else None, + "prev_close_price": ( + str(obj.prev_close_price) if obj.prev_close_price is not None else None + ), "last_price": str(obj.last_price), "last_qty": str(obj.last_qty), "bid_price": str(obj.bid_price), diff --git a/nautilus_trader/adapters/binance/futures/schemas/account.py b/nautilus_trader/adapters/binance/futures/schemas/account.py index 94ac22173249..515378a59b18 100644 --- a/nautilus_trader/adapters/binance/futures/schemas/account.py +++ b/nautilus_trader/adapters/binance/futures/schemas/account.py @@ -86,7 +86,9 @@ class BinanceFuturesAccountInfo(msgspec.Struct, kw_only=True, frozen=True): canDeposit: bool # if can transfer in asset canWithdraw: bool # if can transfer out asset updateTime: int - totalInitialMargin: str | None = None # total initial margin required with current mark price (useless with isolated positions), only for USDT + totalInitialMargin: str | None = ( + None # total initial margin required with current mark price (useless with isolated positions), only for USDT + ) totalMaintMargin: str | None = None # total maintenance margin required, only for USDT asset totalWalletBalance: str | None = None # total wallet balance, only for USDT asset totalUnrealizedProfit: str | None = None # total unrealized profit, only for USDT asset diff --git a/nautilus_trader/adapters/binance/futures/schemas/market.py b/nautilus_trader/adapters/binance/futures/schemas/market.py index 5718bc969194..c78fae4ded63 100644 --- a/nautilus_trader/adapters/binance/futures/schemas/market.py +++ b/nautilus_trader/adapters/binance/futures/schemas/market.py @@ -120,7 +120,9 @@ class BinanceFuturesMarkFunding(msgspec.Struct, frozen=True): symbol: str markPrice: str # Mark price indexPrice: str # Index price - estimatedSettlePrice: str # Estimated Settle Price (only useful in the last hour before the settlement starts) + estimatedSettlePrice: ( + str # Estimated Settle Price (only useful in the last hour before the settlement starts) + ) lastFundingRate: str # This is the lasted funding rate nextFundingTime: int interestRate: str diff --git a/nautilus_trader/adapters/bybit/common/enums.py b/nautilus_trader/adapters/bybit/common/enums.py index 2f6f8326e811..c2cbfd5284ac 100644 --- a/nautilus_trader/adapters/bybit/common/enums.py +++ b/nautilus_trader/adapters/bybit/common/enums.py @@ -209,15 +209,21 @@ def __init__(self) -> None: self.aggregation_kline_mapping = { BarAggregation.MINUTE: lambda x: BybitKlineInterval(f"{x}"), BarAggregation.HOUR: lambda x: BybitKlineInterval(f"{x * 60}"), - BarAggregation.DAY: lambda x: BybitKlineInterval("D") - if x == 1 - else raise_error(ValueError(f"Bybit incorrect day kline interval {x}")), - BarAggregation.WEEK: lambda x: BybitKlineInterval("W") - if x == 1 - else raise_error(ValueError(f"Bybit incorrect week kline interval {x}")), - BarAggregation.MONTH: lambda x: BybitKlineInterval("M") - if x == 1 - else raise_error(ValueError(f"Bybit incorrect month kline interval {x}")), + BarAggregation.DAY: lambda x: ( + BybitKlineInterval("D") + if x == 1 + else raise_error(ValueError(f"Bybit incorrect day kline interval {x}")) + ), + BarAggregation.WEEK: lambda x: ( + BybitKlineInterval("W") + if x == 1 + else raise_error(ValueError(f"Bybit incorrect week kline interval {x}")) + ), + BarAggregation.MONTH: lambda x: ( + BybitKlineInterval("M") + if x == 1 + else raise_error(ValueError(f"Bybit incorrect month kline interval {x}")) + ), } self.valid_order_types = { OrderType.MARKET, diff --git a/nautilus_trader/adapters/interactive_brokers/client/client.py b/nautilus_trader/adapters/interactive_brokers/client/client.py index f5ad74479558..a29358bd0460 100644 --- a/nautilus_trader/adapters/interactive_brokers/client/client.py +++ b/nautilus_trader/adapters/interactive_brokers/client/client.py @@ -486,8 +486,12 @@ async def _run_internal_msg_queue(self) -> None: self._internal_msg_queue.task_done() except asyncio.CancelledError: log_msg = f"Internal message queue processing stopped. (qsize={self._internal_msg_queue.qsize()})." - self._log.warning(log_msg) if not self._internal_msg_queue.empty() else self._log.debug( - log_msg, + ( + self._log.warning(log_msg) + if not self._internal_msg_queue.empty() + else self._log.debug( + log_msg, + ) ) finally: self._eclient.disconnect() diff --git a/nautilus_trader/analysis/statistic.py b/nautilus_trader/analysis/statistic.py index 8a58a502dfca..db6e7b20252f 100644 --- a/nautilus_trader/analysis/statistic.py +++ b/nautilus_trader/analysis/statistic.py @@ -128,7 +128,7 @@ def calculate_from_positions(self, positions: list[Position]) -> Any | None: A JSON serializable primitive. """ - ... # Override in implementation + # Override in implementation def _check_valid_returns(self, returns: pd.Series) -> bool: if returns is None or returns.empty or returns.isna().all(): diff --git a/nautilus_trader/backtest/node.py b/nautilus_trader/backtest/node.py index 46130646e394..d4fe63d63d62 100644 --- a/nautilus_trader/backtest/node.py +++ b/nautilus_trader/backtest/node.py @@ -289,9 +289,9 @@ def _run_streaming( bar_type = None session = catalog.backend_session( data_cls=config.data_type, - instrument_ids=[config.instrument_id] - if config.instrument_id and not bar_type - else [], + instrument_ids=( + [config.instrument_id] if config.instrument_id and not bar_type else [] + ), bar_types=[bar_type] if bar_type else [], start=config.start_time, end=config.end_time, diff --git a/nautilus_trader/core/rust/algorithms.pxd b/nautilus_trader/core/rust/algorithms.pxd index 31f38eb554a9..774675b69162 100644 --- a/nautilus_trader/core/rust/algorithms.pxd +++ b/nautilus_trader/core/rust/algorithms.pxd @@ -2,8 +2,11 @@ from cpython.object cimport PyObject from libc.stdint cimport uint8_t, uint64_t, uintptr_t + from nautilus_trader.core.rust.core cimport CVec, UUID4_t -from nautilus_trader.core.rust.model cimport Symbol_t, Venue_t, InstrumentId_t, Price_t +from nautilus_trader.core.rust.model cimport (InstrumentId_t, Price_t, + Symbol_t, Venue_t) + cdef extern from "../includes/algorithms.h": diff --git a/nautilus_trader/core/rust/model.pyx b/nautilus_trader/core/rust/model.pyx index cbddc04aa8cd..f43eaa98b2fe 100644 --- a/nautilus_trader/core/rust/model.pyx +++ b/nautilus_trader/core/rust/model.pyx @@ -19,13 +19,14 @@ from nautilus_trader.core.rust.model cimport AccountType # type: ignore from nautilus_trader.core.rust.model cimport AggregationSource # type: ignore from nautilus_trader.core.rust.model cimport AggressorSide # type: ignore from nautilus_trader.core.rust.model cimport AssetClass # type: ignore -from nautilus_trader.core.rust.model cimport InstrumentClass # type: ignore from nautilus_trader.core.rust.model cimport BookAction # type: ignore from nautilus_trader.core.rust.model cimport BookType # type: ignore from nautilus_trader.core.rust.model cimport ContingencyType # type: ignore from nautilus_trader.core.rust.model cimport CurrencyType # type: ignore from nautilus_trader.core.rust.model cimport HaltReason # type: ignore -from nautilus_trader.core.rust.model cimport InstrumentCloseType # type: ignore +from nautilus_trader.core.rust.model cimport InstrumentClass # type: ignore +from nautilus_trader.core.rust.model cimport \ + InstrumentCloseType # type: ignore from nautilus_trader.core.rust.model cimport LiquiditySide # type: ignore from nautilus_trader.core.rust.model cimport MarketStatus # type: ignore from nautilus_trader.core.rust.model cimport OmsType # type: ignore diff --git a/nautilus_trader/data/client.pyx b/nautilus_trader/data/client.pyx index afef9d5d47e5..c8ec15a8124c 100644 --- a/nautilus_trader/data/client.pyx +++ b/nautilus_trader/data/client.pyx @@ -14,6 +14,7 @@ # ------------------------------------------------------------------------------------------------- from nautilus_trader.config.common import NautilusConfig + from cpython.datetime cimport datetime from nautilus_trader.cache.cache cimport Cache diff --git a/nautilus_trader/persistence/catalog/parquet.py b/nautilus_trader/persistence/catalog/parquet.py index 471d4ab6c19a..fc2dd17770de 100644 --- a/nautilus_trader/persistence/catalog/parquet.py +++ b/nautilus_trader/persistence/catalog/parquet.py @@ -15,6 +15,7 @@ from __future__ import annotations +import itertools import os import pathlib import platform @@ -672,7 +673,7 @@ def _read_feather( if raise_on_failed_deserialize: raise print(f"Failed to deserialize {cls_name}: {e}") - return sorted(sum(data.values(), []), key=lambda x: x.ts_init) + return sorted(itertools.chain.from_iterable(data.values()), key=lambda x: x.ts_init) def _list_feather_files( self, diff --git a/nautilus_trader/system/kernel.py b/nautilus_trader/system/kernel.py index 995038eca99d..090604956cec 100644 --- a/nautilus_trader/system/kernel.py +++ b/nautilus_trader/system/kernel.py @@ -167,9 +167,11 @@ def __init__( # noqa (too complex) machine_id=self._machine_id, instance_id=self._instance_id, level_stdout=log_level_from_str(logging.log_level), - level_file=log_level_from_str(logging.log_level_file) - if logging.log_level_file is not None - else LogLevel.OFF, + level_file=( + log_level_from_str(logging.log_level_file) + if logging.log_level_file is not None + else LogLevel.OFF + ), directory=logging.log_directory, file_name=logging.log_file_name, file_format=logging.log_file_format, diff --git a/poetry.lock b/poetry.lock index 96b0ce7ff1a9..17d133ffe513 100644 --- a/poetry.lock +++ b/poetry.lock @@ -248,13 +248,13 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2023.11.17" +version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, - {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] [[package]] @@ -464,7 +464,7 @@ name = "css-html-js-minify" version = "2.5.5" description = "CSS HTML JS Minifier" optional = false -python-versions = ">=3.6" +python-versions = "*" files = [ {file = "css-html-js-minify-2.5.5.zip", hash = "sha256:4a9f11f7e0496f5284d12111f3ba4ff5ff2023d12f15d195c9c48bd97013746c"}, {file = "css_html_js_minify-2.5.5-py2.py3-none-any.whl", hash = "sha256:3da9d35ac0db8ca648c1b543e0e801d7ca0bab9e6bfd8418fee59d5ae001727a"}, @@ -1124,85 +1124,101 @@ yaml = ["pyyaml"] [[package]] name = "multidict" -version = "6.0.4" +version = "6.0.5" description = "multidict implementation" optional = false python-versions = ">=3.7" files = [ - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, - {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, - {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, - {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, - {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, - {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, - {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, - {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, - {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, - {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, - {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, - {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, - {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, ] [[package]] diff --git a/tests/integration_tests/adapters/betfair/test_betfair_execution.py b/tests/integration_tests/adapters/betfair/test_betfair_execution.py index de338be8b199..3ee1be5c14b5 100644 --- a/tests/integration_tests/adapters/betfair/test_betfair_execution.py +++ b/tests/integration_tests/adapters/betfair/test_betfair_execution.py @@ -105,9 +105,9 @@ async def _setup_order_state( price=betfair_float_to_price(order_update.p), client_order_id=client_order_id, ) - exec_client.venue_order_id_to_client_order_id[ - venue_order_id - ] = client_order_id + exec_client.venue_order_id_to_client_order_id[venue_order_id] = ( + client_order_id + ) await _accept_order(order, venue_order_id, exec_client, strategy, cache) if include_fills and order_update.sm: diff --git a/tests/integration_tests/adapters/betfair/test_kit.py b/tests/integration_tests/adapters/betfair/test_kit.py index 16e536a1a5f3..3df614233318 100644 --- a/tests/integration_tests/adapters/betfair/test_kit.py +++ b/tests/integration_tests/adapters/betfair/test_kit.py @@ -227,25 +227,29 @@ def betfair_backtest_run_config( bypass_logging=bypass_logging, ), risk_engine=RiskEngineConfig(bypass=bypass_risk), - streaming=BetfairTestStubs.streaming_config( - catalog_fs_protocol=catalog_fs_protocol, - catalog_path=catalog_path, - flush_interval_ms=flush_interval_ms, - ) - if persist - else None, - strategies=[ - ImportableStrategyConfig( - strategy_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalance", - config_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalanceConfig", - config={ - "instrument_id": instrument_id.value, - "max_trade_size": 50, - }, - ), - ] - if add_strategy - else None, + streaming=( + BetfairTestStubs.streaming_config( + catalog_fs_protocol=catalog_fs_protocol, + catalog_path=catalog_path, + flush_interval_ms=flush_interval_ms, + ) + if persist + else None + ), + strategies=( + [ + ImportableStrategyConfig( + strategy_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalance", + config_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalanceConfig", + config={ + "instrument_id": instrument_id.value, + "max_trade_size": 50, + }, + ), + ] + if add_strategy + else None + ), ) run_config = BacktestRunConfig( # typing: ignore engine=engine_config, diff --git a/tests/integration_tests/live/test_live_node.py b/tests/integration_tests/live/test_live_node.py index facd3ae7ca8e..0ed7e6d0bbad 100644 --- a/tests/integration_tests/live/test_live_node.py +++ b/tests/integration_tests/live/test_live_node.py @@ -178,12 +178,12 @@ def test_node_build_objects(self, monkeypatch): # Mock factories so nothing actually connects from nautilus_trader.adapters.binance import factories - mock_data_factory = ( - factories.BinanceLiveDataClientFactory.create - ) = unittest.mock.MagicMock() - mock_exec_factory = ( - factories.BinanceLiveExecClientFactory.create - ) = unittest.mock.MagicMock() + mock_data_factory = factories.BinanceLiveDataClientFactory.create = ( + unittest.mock.MagicMock() + ) + mock_exec_factory = factories.BinanceLiveExecClientFactory.create = ( + unittest.mock.MagicMock() + ) # Act - lazy way of mocking the whole client with pytest.raises(TypeError): From 2f49dc3397e8829011fc5675d9f18ea2dc43bbb2 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 2 Feb 2024 21:39:18 +1100 Subject: [PATCH 42/42] Update release notes --- RELEASES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 6d5dba0215a2..c6a29ad21bdd 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,6 +1,6 @@ # NautilusTrader 1.186.0 Beta -Released on TBD (UTC). +Released on 2nd February 2024 (UTC). ### Enhancements None @@ -11,6 +11,8 @@ None ### Fixes - Fixed Interactive Brokers get account positions bug (#1475), thanks @benjaminsingleton - Fixed `TimeBarAggregator` handling of interval types on build +- Fixed `BinanceSpotExecutionClient` non-existent method name, thanks @sunlei +- Fixed unused `psutil` import, thanks @sunlei ---