Merge pull request #81 from peaqnetwork/feat/1208894234090825_fee_wei…

…ght_collection

Feat/1208894234090825 fee weight collection

.gitignore

@@ -171,3 +171,4 @@ Sessionx.vim
 tags
 # Persistent undo
 [._]*.un~
+reports/

tests/bridge_asset_factory_test.py

@@ -10,7 +10,7 @@
 from tools.peaq_eth_utils import calculate_asset_to_evm_address
 from tools.peaq_eth_utils import get_eth_info
 from tools.constants import KP_GLOBAL_SUDO
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from web3 import Web3
 
 
@@ -20,7 +20,7 @@
 BATCH_ABI_FILE = 'ETH/batch/abi'
 BATCH_ADDRESS = '0x0000000000000000000000000000000000000805'
 
-IERC20PLUS_ABI_FILE = 'ETH/erc20/plus.abi'
+IERC20PLUS_ABI_FILE = 'ETH/erc20plus/abi'
 
 
 @pytest.mark.eth

tests/bridge_balance_erc20_test.py

@@ -5,7 +5,7 @@
 from peaq.utils import ExtrinsicBatch
 from peaq.utils import get_account_balance
 from tools.utils import get_modified_chain_spec
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.constants import WS_URL, ETH_URL
 from tools.constants import KP_GLOBAL_SUDO
 from tools.peaq_eth_utils import get_contract

tests/bridge_batch_test.py

@@ -10,7 +10,7 @@
 from peaq.utils import ExtrinsicBatch
 from web3 import Web3
 from tools.constants import KP_GLOBAL_SUDO
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.peaq_eth_utils import generate_random_hex
 
 
@@ -22,10 +22,10 @@
 ABI_FILE = 'ETH/batch/abi'
 BATCH_ADDRESS = '0x0000000000000000000000000000000000000805'
 
-DID_ABI_FILE = 'ETH/did/did.sol.json'
+DID_ABI_FILE = 'ETH/did/abi'
 DID_ADDRESS = '0x0000000000000000000000000000000000000800'
 
-STORAGE_ABI_FILE = 'ETH/storage/storage.sol.json'
+STORAGE_ABI_FILE = 'ETH/storage/abi'
 STORAGE_ADDRESS = '0x0000000000000000000000000000000000000801'
 
 

tests/bridge_did_test.py

@@ -6,7 +6,7 @@
 from peaq.extrinsic import transfer
 from tools.peaq_eth_utils import call_eth_transfer_a_lot, get_contract, generate_random_hex
 from tools.constants import WS_URL, ETH_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.peaq_eth_utils import get_eth_chain_id
 from web3 import Web3
 
@@ -21,7 +21,7 @@
 DID_ADDRESS = '0x0000000000000000000000000000000000000800'
 ETH_PRIVATE_KEY = '0xa2899b053679427c8c446dc990c8990c75052fd3009e563c6a613d982d6842fe'
 VALIDITY = 1000
-ABI_FILE = 'ETH/did/did.sol.json'
+ABI_FILE = 'ETH/did/abi'
 
 
 @pytest.mark.eth

tests/bridge_erc20_test.py

@@ -3,7 +3,7 @@
 from substrateinterface import SubstrateInterface, Keypair
 from tools.asset import batch_create_asset, get_valid_asset_id, batch_set_metadata, batch_mint
 from tools.constants import WS_URL, ETH_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from peaq.utils import ExtrinsicBatch
 from tools.peaq_eth_utils import get_contract
 from tools.peaq_eth_utils import get_eth_chain_id

tests/bridge_multiple_collator_test.py

@@ -4,7 +4,7 @@
 from tools.runtime_upgrade import wait_until_block_height
 from substrateinterface import SubstrateInterface, Keypair
 from tools.constants import WS_URL, ETH_URL, RELAYCHAIN_WS_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from peaq.utils import ExtrinsicBatch
 from tools.peaq_eth_utils import get_contract
 from tools.peaq_eth_utils import get_eth_chain_id

tests/bridge_parachain_staking_test.py

@@ -4,7 +4,7 @@
 from tools.runtime_upgrade import wait_until_block_height
 from substrateinterface import SubstrateInterface, Keypair
 from tools.constants import WS_URL, ETH_URL, RELAYCHAIN_WS_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from peaq.utils import ExtrinsicBatch
 from tests import utils_func as TestUtils
 from tools.peaq_eth_utils import get_contract

tests/bridge_rbac_test.py

@@ -1,7 +1,7 @@
 import pytest
 
 from substrateinterface import SubstrateInterface, Keypair, KeypairType
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.constants import WS_URL, ETH_URL
 from peaq.eth import calculate_evm_account, calculate_evm_addr
 from tools.peaq_eth_utils import get_eth_chain_id
@@ -22,7 +22,7 @@
 # H160 Address to use for EVM transactions
 ETH_PRIVATE_KEY = generate_random_hex(15).encode("utf-8")
 # RBAC Precompile ABI
-ABI_FILE = 'ETH/rbac/rbac.sol.json'
+ABI_FILE = 'ETH/rbac/abi'
 # Number of tokens with decimals
 TOKEN_NUM = 10000 * pow(10, 15)
 

tests/bridge_storage_test.py

@@ -4,7 +4,7 @@
 from peaq.eth import calculate_evm_account_hex, calculate_evm_addr, calculate_evm_account
 from peaq.extrinsic import transfer
 from tools.constants import WS_URL, ETH_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.peaq_eth_utils import get_eth_chain_id
 from tools.peaq_eth_utils import call_eth_transfer_a_lot, get_contract, generate_random_hex
 from tools.peaq_eth_utils import TX_SUCCESS_STATUS
@@ -24,7 +24,7 @@
 KP_SRC = Keypair.create_from_uri('//Alice')
 STORAGE_ADDRESS = '0x0000000000000000000000000000000000000801'
 ETH_PRIVATE_KEY = '0xa2899b053679427c8c446dc990c8990c75052fd3009e563c6a613d982d6842fe'
-ABI_FILE = 'ETH/storage/storage.sol.json'
+ABI_FILE = 'ETH/storage/abi'
 TOKEN_NUM = 10000 * pow(10, 15)
 
 

tests/bridge_vest_test.py

@@ -9,7 +9,7 @@
 from tools.constants import KP_GLOBAL_SUDO
 from tools.constants import WS_URL, ETH_URL
 from tools.utils import get_account_balance_locked
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.peaq_eth_utils import get_eth_chain_id
 from tools.peaq_eth_utils import get_eth_info
 from tools.utils import batch_fund

tests/bridge_xcmutils_test.py

@@ -10,7 +10,7 @@
 from peaq.utils import ExtrinsicBatch
 from web3 import Web3
 from tools.constants import KP_GLOBAL_SUDO
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from peaq.utils import get_account_balance
 from tests import utils_func as TestUtils
 from tools.asset import wait_for_account_asset_change_wrap

tests/bridge_xtokens_test.py

@@ -8,7 +8,7 @@
 from substrateinterface import SubstrateInterface, Keypair
 from tools.constants import ETH_URL, RELAYCHAIN_WS_URL
 from tools.constants import WS_URL, ACA_WS_URL, PARACHAIN_WS_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from peaq.utils import get_account_balance
 from peaq.utils import ExtrinsicBatch
 from tools.asset import convert_enum_to_asset_id

tests/conftest.py

@@ -2,6 +2,12 @@
 import importlib
 import sys
 import tools.utils  # noqa: F401
+from tests.substrate_utils import monkey_submit_extrinsic_for_fee_weight
+from tests.substrate_utils import generate_substrate_weight_fee_report
+from tests.evm_utils import generate_evm_fee_report
+from substrateinterface import SubstrateInterface
+
+SubstrateInterface.submit_extrinsic = monkey_submit_extrinsic_for_fee_weight
 
 
 def pytest_runtest_setup(item):
@@ -11,3 +17,8 @@ def pytest_runtest_setup(item):
         importlib.reload(sys.modules['substrateinterface'])
     if 'peaq.utils' in sys.modules:
         importlib.reload(sys.modules['peaq.utils'])
+
+
+def pytest_sessionfinish(session, exitstatus):
+    generate_substrate_weight_fee_report()
+    generate_evm_fee_report()

tests/evm_eth_rpc_test.py

@@ -7,7 +7,7 @@
 from peaq.utils import ExtrinsicBatch
 from tools.constants import KP_GLOBAL_SUDO
 from tools.constants import WS_URL, ETH_URL
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.peaq_eth_utils import get_eth_chain_id
 from tools.peaq_eth_utils import deploy_contract
 from tools.peaq_eth_utils import call_eth_transfer_a_lot

tests/evm_utils.py

@@ -0,0 +1,116 @@
+from tools import peaq_eth_utils as PeaqEthUtils
+from eth_utils import function_abi_to_4byte_selector
+import os
+import json
+import datetime
+from tools.constants import PARACHAIN_ETH_URL
+
+evm_all_fee_data = {}
+func_selector_dict = {}
+
+
+def load_all_function(folder, abi):
+    function_dict = {}
+    for item in abi:
+        if item["type"] != "function":
+            continue
+        selector = f'0x{function_abi_to_4byte_selector(item).hex()}'
+        function_dict[selector] = f'{folder}.{item["name"]}'
+    return function_dict
+
+
+def load_all_abi():
+    # traverse ETH/*/abi files
+    folders = os.listdir("ETH")
+    for folder in folders:
+        if folder == ".pytest_cache":
+            continue
+        with open(os.path.join('ETH', folder, 'abi')) as f:
+            abi = json.load(f)
+        for k, v in load_all_function(folder, abi).items():
+            func_selector_dict[k] = v
+
+
+def compose_batch_function_name(w3, input_data):
+    batch_abi = json.load(open('ETH/batch/abi'))
+
+    batch = w3.eth.contract(address='0x0000000000000000000000000000000000000805', abi=batch_abi)
+    func_name, args = batch.decode_function_input(input_data)
+    inner_func_names = []
+    for call_data in args['callData']:
+        function_selector = "0x" + call_data.hex()[:8]
+        if function_selector not in func_selector_dict:
+            inner_func_names.append(f'Unknown.{function_selector}')
+        else:
+            inner_func_names.append(func_selector_dict[function_selector])
+    return f"({','.join(inner_func_names)})"
+
+
+def compose_function_name(w3, tx):
+    if 'data' not in tx:
+        return 'Transfer'
+    input_data = tx['data']
+    function_selector = input_data[:10]
+    if function_selector not in func_selector_dict:
+        import pdb
+        pdb.set_trace()
+        return f'Unknown.{function_selector}'
+
+    name = func_selector_dict[function_selector]
+    if name.startswith('batch'):
+        inner_name = compose_batch_function_name(w3, input_data)
+        name += inner_name
+
+    return name
+
+
+def sign_and_submit_evm_transaction(tx, w3, signer):
+    receipt = PeaqEthUtils.sign_and_submit_evm_transaction(tx, w3, signer)
+    gas_used = receipt['gasUsed']
+    effective_gas_price = receipt['effectiveGasPrice']
+
+    if w3.provider.endpoint_uri != PARACHAIN_ETH_URL:
+        return receipt
+
+    if not func_selector_dict:
+        load_all_abi()
+
+    name = compose_function_name(w3, tx)
+    fee = int(w3.from_wei(gas_used * effective_gas_price, 'wei'))
+    if name not in evm_all_fee_data:
+        evm_all_fee_data[name] = []
+    evm_all_fee_data[name].append({
+        'fee': fee,
+    })
+
+    return receipt
+
+
+def generate_evm_fee_report():
+    # get date by format "YYYY-MM-DD-HH-MM"
+    now = datetime.datetime.now()
+    date = now.strftime("%Y-%m-%d-%H-%M")
+
+    folder = "reports"
+    report_file = f"evm_fee_summary_{date}.json"
+
+    if not os.path.exists(folder):
+        os.makedirs(folder)
+
+    report_path = os.path.join(folder, report_file)
+    summary_data = process_fee_data()
+    with open(report_path, "w") as f:
+        json.dump(summary_data, f, indent=4)
+
+    print('')
+    print(f"EVM fee data saved to {report_path}")
+
+
+def process_fee_data():
+    summary_data = {}
+    for extrinsic_name, data in evm_all_fee_data.items():
+        summary_data[extrinsic_name] = {
+            'fee': sum([d['fee'] for d in data]) / len(data),
+            'len': len(data)
+        }
+    return summary_data

tests/pallet_address_unification.py

@@ -3,7 +3,7 @@
 from substrateinterface import SubstrateInterface, Keypair, KeypairType
 from tools.constants import WS_URL, ETH_URL
 from tools.constants import KP_GLOBAL_SUDO
-from tools.peaq_eth_utils import sign_and_submit_evm_transaction
+from tests.evm_utils import sign_and_submit_evm_transaction
 from tools.asset import batch_create_asset, batch_mint, get_valid_asset_id
 from tools.asset import get_asset_balance
 from tools.evm_claim_sign import calculate_claim_signature, claim_account