From 6330f9571c8fd34b8ffedd1f1b6a01d64352ebf8 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Fri, 18 Oct 2024 16:54:32 +0800 Subject: [PATCH 1/5] Problem: cometbft rpc is more efficient to broadcast tx --- CHANGELOG.md | 1 + testground/benchmark/benchmark/cosmostx.py | 86 +++++++++++++++++++ testground/benchmark/benchmark/peer.py | 2 +- testground/benchmark/benchmark/transaction.py | 46 ++++++++-- testground/benchmark/benchmark/utils.py | 1 + testground/benchmark/overlay.nix | 1 + testground/benchmark/poetry.lock | 12 ++- testground/benchmark/pyproject.toml | 1 + 8 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 testground/benchmark/benchmark/cosmostx.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2888c085be..c60c9b3b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ * [#1645](https://github.com/crypto-org-chain/cronos/pull/1645) Gen test tx in parallel even in single node. * (testground)[#1644](https://github.com/crypto-org-chain/cronos/pull/1644) load generator retry with backoff on error. * [#1648](https://github.com/crypto-org-chain/cronos/pull/1648) Add abort OE in PrepareProposal. +* (testground)[#]() Benchmark use cosmos broadcast rpc. *Oct 14, 2024* diff --git a/testground/benchmark/benchmark/cosmostx.py b/testground/benchmark/benchmark/cosmostx.py new file mode 100644 index 0000000000..2177ced51c --- /dev/null +++ b/testground/benchmark/benchmark/cosmostx.py @@ -0,0 +1,86 @@ +from typing import Optional + +from cprotobuf import Field, ProtoEntity + + +class Any(ProtoEntity): + type_url = Field("string", 1) + value = Field("bytes", 2) + + +def build_any(type_url: str, msg: Optional[ProtoEntity] = None) -> Any: + value = b"" + if msg is not None: + value = msg.SerializeToString() + return Any(type_url=type_url, value=value) + + +class TxBody(ProtoEntity): + messages = Field(Any, 1, repeated=True) + memo = Field("string", 2) + timeout_height = Field("uint64", 3) + extension_options = Field(Any, 1023, repeated=True) + non_critical_extension_options = Field(Any, 2047, repeated=True) + + +class CompactBitArray(ProtoEntity): + extra_bits_stored = Field("uint32", 1) + elems = Field("bytes", 2) + + +class ModeInfoSingle(ProtoEntity): + mode = Field("int32", 1) + + +class ModeInfoMulti(ProtoEntity): + bitarray = Field(CompactBitArray, 1) + mode_infos = Field("ModeInfo", 2, repeated=True) + + +class ModeInfo(ProtoEntity): + single = Field("ModeInfo.Single", 1) + multi = Field("ModeInfo.Multi", 2) + + +class SignerInfo(ProtoEntity): + public_key = Field(Any, 1) + mode_info = Field(ModeInfo, 2) + sequence = Field("uint64", 3) + + +class Coin(ProtoEntity): + denom = Field("string", 1) + amount = Field("string", 2) + + +class Fee(ProtoEntity): + amount = Field(Coin, 1, repeated=True) + gas_limit = Field("uint64", 2) + payer = Field("string", 3) + granter = Field("string", 4) + + +class Tip(ProtoEntity): + amount = Field(Coin, 1, repeated=True) + tipper = Field("string", 2) + + +class AuthInfo(ProtoEntity): + signer_infos = Field(SignerInfo, 1, repeated=True) + fee = Field(Fee, 2) + tip = Field(Tip, 3) + + +class TxRaw(ProtoEntity): + body = Field("bytes", 1) + auth_info = Field("bytes", 2) + signatures = Field("bytes", 3, repeated=True) + + +class MsgEthereumTx(ProtoEntity): + from_ = Field("bytes", 5) + raw = Field("bytes", 6) + + +class ExtensionOptionsEthereumTx(ProtoEntity): + pass diff --git a/testground/benchmark/benchmark/peer.py b/testground/benchmark/benchmark/peer.py index 83da764961..a1526f8870 100644 --- a/testground/benchmark/benchmark/peer.py +++ b/testground/benchmark/benchmark/peer.py @@ -11,6 +11,7 @@ from .cli import ChainCommand from .types import Balance, GenesisAccount, PeerPacket from .utils import ( + DEFAULT_DENOM, bech32_to_eth, eth_to_bech32, gen_account, @@ -19,7 +20,6 @@ patch_toml, ) -DEFAULT_DENOM = "basecro" VAL_ACCOUNT = "validator" VAL_INITIAL_AMOUNT = Balance(amount="100000000000000000000", denom=DEFAULT_DENOM) VAL_STAKED_AMOUNT = Balance(amount="10000000000000000000", denom=DEFAULT_DENOM) diff --git a/testground/benchmark/benchmark/transaction.py b/testground/benchmark/benchmark/transaction.py index 78c77a3804..48d8685e7e 100644 --- a/testground/benchmark/benchmark/transaction.py +++ b/testground/benchmark/benchmark/transaction.py @@ -1,4 +1,5 @@ import asyncio +import base64 import itertools import multiprocessing import os @@ -9,9 +10,11 @@ import backoff import eth_abi import ujson +from hexbytes import HexBytes +from . import cosmostx from .erc20 import CONTRACT_ADDRESS -from .utils import LOCAL_JSON_RPC, gen_account, split +from .utils import DEFAULT_DENOM, LOCAL_RPC, gen_account, split GAS_PRICE = 1000000000 CHAIN_ID = 777 @@ -63,7 +66,9 @@ def _do_job(job: Job): for acct in accounts: txs = [] for i in range(job.num_txs): - txs.append(acct.sign_transaction(job.create_tx(i)).rawTransaction.hex()) + tx = job.create_tx(i) + raw = acct.sign_transaction(tx).rawTransaction + txs.append(build_cosmos_tx(tx, raw, HexBytes(acct.address))) total += 1 if total % 1000 == 0: print("generated", total, "txs for node", job.global_seq) @@ -107,15 +112,46 @@ def load(datadir: Path, global_seq: int) -> [str]: return ujson.load(f) +def build_cosmos_tx(tx: dict, raw: bytes, sender: bytes) -> str: + """ + return base64 encoded cosmos tx + """ + msg = cosmostx.build_any( + "/ethermint.evm.v1.MsgEthereumTx", + cosmostx.MsgEthereumTx( + from_=sender, + raw=raw, + ), + ) + fee = tx["gas"] * tx["gasPrice"] + body = cosmostx.TxBody( + messages=[msg], + extension_options=[ + cosmostx.build_any("/ethermint.evm.v1.ExtensionOptionsEthereumTx") + ], + ) + auth_info = cosmostx.AuthInfo( + fee=cosmostx.Fee( + amount=[cosmostx.Coin(denom=DEFAULT_DENOM, amount=str(fee))], + gas_limit=tx["gas"], + ) + ) + return base64.b64encode( + cosmostx.TxRaw( + body=body.SerializeToString(), auth_info=auth_info.SerializeToString() + ).SerializeToString() + ).decode() + + @backoff.on_predicate(backoff.expo, max_time=60, max_value=5) @backoff.on_exception(backoff.expo, aiohttp.ClientError, max_time=60, max_value=5) async def async_sendtx(session, raw): async with session.post( - LOCAL_JSON_RPC, + LOCAL_RPC, json={ "jsonrpc": "2.0", - "method": "eth_sendRawTransaction", - "params": [raw], + "method": "broadcast_tx_async", + "params": {"tx": raw}, "id": 1, }, ) as rsp: diff --git a/testground/benchmark/benchmark/utils.py b/testground/benchmark/benchmark/utils.py index 5a0c033c98..e3c8e94e32 100644 --- a/testground/benchmark/benchmark/utils.py +++ b/testground/benchmark/benchmark/utils.py @@ -12,6 +12,7 @@ from hexbytes import HexBytes from web3._utils.transactions import fill_nonce, fill_transaction_defaults +DEFAULT_DENOM = "basecro" CRONOS_ADDRESS_PREFIX = "crc" LOCAL_RPC = "http://127.0.0.1:26657" LOCAL_JSON_RPC = "http://127.0.0.1:8545" diff --git a/testground/benchmark/overlay.nix b/testground/benchmark/overlay.nix index 280f3b46cb..d1aeb3e7dd 100644 --- a/testground/benchmark/overlay.nix +++ b/testground/benchmark/overlay.nix @@ -10,6 +10,7 @@ let docker = [ "hatchling" "hatch-vcs" ]; pyunormalize = [ "setuptools" ]; pytest-github-actions-annotate-failures = [ "setuptools" ]; + cprotobuf = [ "setuptools" ]; }; in lib.mapAttrs diff --git a/testground/benchmark/poetry.lock b/testground/benchmark/poetry.lock index 386a590fd5..72b5b389a8 100644 --- a/testground/benchmark/poetry.lock +++ b/testground/benchmark/poetry.lock @@ -533,6 +533,16 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cprotobuf" +version = "0.1.11" +description = "pythonic and high performance protocol buffer implementation." +optional = false +python-versions = "*" +files = [ + {file = "cprotobuf-0.1.11.tar.gz", hash = "sha256:d2d88c8de840275205e64e530052c653dd25a0fb9e5cd9f7e39ce8f762d7c0a4"}, +] + [[package]] name = "cytoolz" version = "0.12.3" @@ -2227,4 +2237,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "777a8252968a616738411c9418f1621f68befb12ee7147ea7ab36918a5a9a118" +content-hash = "bebd65d7cba833c0079907f3174c7441312f4daa9c254a5030bdeb7c81bf6387" diff --git a/testground/benchmark/pyproject.toml b/testground/benchmark/pyproject.toml index f07b96db2f..c18f9132d6 100644 --- a/testground/benchmark/pyproject.toml +++ b/testground/benchmark/pyproject.toml @@ -20,6 +20,7 @@ ujson = "^5.10.0" jsonmerge = "^1.9.2" backoff = "^2.2.1" +cprotobuf = "^0.1.11" [tool.poetry.group.dev.dependencies] pytest = "^8.2" pytest-github-actions-annotate-failures = "^0.2.0" From 57fd9d6029cf1cc99c757c6cb9ec9dd8ca7e23cf Mon Sep 17 00:00:00 2001 From: yihuang Date: Fri, 18 Oct 2024 17:00:27 +0800 Subject: [PATCH 2/5] Update CHANGELOG.md Signed-off-by: yihuang --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c60c9b3b2e..735a7d0261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ * [#1645](https://github.com/crypto-org-chain/cronos/pull/1645) Gen test tx in parallel even in single node. * (testground)[#1644](https://github.com/crypto-org-chain/cronos/pull/1644) load generator retry with backoff on error. * [#1648](https://github.com/crypto-org-chain/cronos/pull/1648) Add abort OE in PrepareProposal. -* (testground)[#]() Benchmark use cosmos broadcast rpc. +* (testground)[#1651](https://github.com/crypto-org-chain/cronos/pull/1651) Benchmark use cosmos broadcast rpc. *Oct 14, 2024* From 9513fa84fcc87ced093e2fa8dc58f2cc95849b8f Mon Sep 17 00:00:00 2001 From: yihuang Date: Fri, 18 Oct 2024 17:09:28 +0800 Subject: [PATCH 3/5] Update testground/benchmark/benchmark/cosmostx.py Signed-off-by: yihuang --- testground/benchmark/benchmark/cosmostx.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testground/benchmark/benchmark/cosmostx.py b/testground/benchmark/benchmark/cosmostx.py index 2177ced51c..ba7eb07d25 100644 --- a/testground/benchmark/benchmark/cosmostx.py +++ b/testground/benchmark/benchmark/cosmostx.py @@ -80,7 +80,3 @@ class TxRaw(ProtoEntity): class MsgEthereumTx(ProtoEntity): from_ = Field("bytes", 5) raw = Field("bytes", 6) - - -class ExtensionOptionsEthereumTx(ProtoEntity): - pass From 393a0aa086480c2ddc668ff8f803b6a7ed765511 Mon Sep 17 00:00:00 2001 From: yihuang Date: Fri, 18 Oct 2024 17:10:02 +0800 Subject: [PATCH 4/5] Update testground/benchmark/benchmark/cosmostx.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: yihuang --- testground/benchmark/benchmark/cosmostx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testground/benchmark/benchmark/cosmostx.py b/testground/benchmark/benchmark/cosmostx.py index ba7eb07d25..587aa84742 100644 --- a/testground/benchmark/benchmark/cosmostx.py +++ b/testground/benchmark/benchmark/cosmostx.py @@ -38,8 +38,8 @@ class ModeInfoMulti(ProtoEntity): class ModeInfo(ProtoEntity): - single = Field("ModeInfo.Single", 1) - multi = Field("ModeInfo.Multi", 2) + single = Field(ModeInfoSingle, 1) + multi = Field(ModeInfoMulti, 2) class SignerInfo(ProtoEntity): From 6b307daddec1904ca4a5bf1a21f4dd1ce76d35bb Mon Sep 17 00:00:00 2001 From: HuangYi Date: Fri, 18 Oct 2024 17:11:15 +0800 Subject: [PATCH 5/5] renmae --- testground/benchmark/benchmark/cosmostx.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/testground/benchmark/benchmark/cosmostx.py b/testground/benchmark/benchmark/cosmostx.py index 587aa84742..bdd47734b7 100644 --- a/testground/benchmark/benchmark/cosmostx.py +++ b/testground/benchmark/benchmark/cosmostx.py @@ -3,24 +3,24 @@ from cprotobuf import Field, ProtoEntity -class Any(ProtoEntity): +class ProtoAny(ProtoEntity): type_url = Field("string", 1) value = Field("bytes", 2) -def build_any(type_url: str, msg: Optional[ProtoEntity] = None) -> Any: +def build_any(type_url: str, msg: Optional[ProtoEntity] = None) -> ProtoAny: value = b"" if msg is not None: value = msg.SerializeToString() - return Any(type_url=type_url, value=value) + return ProtoAny(type_url=type_url, value=value) class TxBody(ProtoEntity): - messages = Field(Any, 1, repeated=True) + messages = Field(ProtoAny, 1, repeated=True) memo = Field("string", 2) timeout_height = Field("uint64", 3) - extension_options = Field(Any, 1023, repeated=True) - non_critical_extension_options = Field(Any, 2047, repeated=True) + extension_options = Field(ProtoAny, 1023, repeated=True) + non_critical_extension_options = Field(ProtoAny, 2047, repeated=True) class CompactBitArray(ProtoEntity): @@ -43,7 +43,7 @@ class ModeInfo(ProtoEntity): class SignerInfo(ProtoEntity): - public_key = Field(Any, 1) + public_key = Field(ProtoAny, 1) mode_info = Field(ModeInfo, 2) sequence = Field("uint64", 3)