From fe0032111acc73ad9a2c2c0d6af2dac5ad9ce08c Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Fri, 18 Apr 2025 20:31:02 +0000 Subject: [PATCH 01/13] chore: refactor out _apply_request_formatters --- tests/core/method-class/test_method.py | 7 +------ web3/_utils/method_formatters.py | 2 +- web3/method.py | 27 ++++++-------------------- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/tests/core/method-class/test_method.py b/tests/core/method-class/test_method.py index 0d82a6b602..88766d9d09 100644 --- a/tests/core/method-class/test_method.py +++ b/tests/core/method-class/test_method.py @@ -17,7 +17,6 @@ ) from web3.method import ( Method, - _apply_request_formatters, default_root_munger, ) from web3.module import ( @@ -61,11 +60,7 @@ def test_get_formatters_default_formatter_for_falsy_config(): default_result_formatters = method.result_formatters( method.method_selector_fn(), "some module" ) - assert _apply_request_formatters(["a", "b", "c"], default_request_formatters) == ( - "a", - "b", - "c", - ) + assert default_request_formatters(["a", "b", "c"]) == ("a", "b", "c") assert apply_result_formatters(default_result_formatters, ["a", "b", "c"]) == [ "a", "b", diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py index 71f27e1970..0136b6d30b 100644 --- a/web3/_utils/method_formatters.py +++ b/web3/_utils/method_formatters.py @@ -1069,7 +1069,7 @@ def get_request_formatters( PYTHONIC_REQUEST_FORMATTERS, ) formatters = combine_formatters(request_formatter_maps, method_name) - return compose(*formatters) + return compose(tuple, *formatters) def raise_block_not_found(params: Tuple[BlockIdentifier, bool]) -> NoReturn: diff --git a/web3/method.py b/web3/method.py index f67828f964..16466aeb5d 100644 --- a/web3/method.py +++ b/web3/method.py @@ -14,13 +14,6 @@ ) import warnings -from eth_utils.curried import ( - to_tuple, -) -from eth_utils.toolz import ( - pipe, -) - from web3._utils.batching import ( RPC_METHODS_UNSUPPORTED_DURING_BATCH, ) @@ -56,16 +49,6 @@ Munger = Callable[..., Any] -@to_tuple -def _apply_request_formatters( - params: Any, request_formatters: Dict[RPCEndpoint, Callable[..., TReturn]] -) -> Tuple[Any, ...]: - if request_formatters: - formatted_params = pipe(params, request_formatters) - return formatted_params - return params - - def _set_mungers( mungers: Optional[Sequence[Munger]], is_property: bool ) -> Sequence[Any]: @@ -232,10 +215,12 @@ def process_params( get_error_formatters(method), self.null_result_formatters(method), ) - request = ( - method, - _apply_request_formatters(params, self.request_formatters(method)), - ) + + if request_formatters := self.request_formatters(method): + params = tuple(request_formatters(params)) # type: ignore[assignment] + + request = method, params + return request, response_formatters From 4742932968eb2f82c1b7315af153be218ed4e34e Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:31:02 +0000 Subject: [PATCH 02/13] chore: refactor --- web3/method.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/method.py b/web3/method.py index 16466aeb5d..645901a670 100644 --- a/web3/method.py +++ b/web3/method.py @@ -217,7 +217,7 @@ def process_params( ) if request_formatters := self.request_formatters(method): - params = tuple(request_formatters(params)) # type: ignore[assignment] + params = request_formatters(params) request = method, params From 23208d10907a7ded080c7622494f9bfaf62f5e79 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 03/13] feat: optimize apply_abi_formatters_to_dict --- web3/_utils/rpc_abi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web3/_utils/rpc_abi.py b/web3/_utils/rpc_abi.py index c401309c55..8220a4a428 100644 --- a/web3/_utils/rpc_abi.py +++ b/web3/_utils/rpc_abi.py @@ -218,8 +218,9 @@ def apply_abi_formatters_to_dict( [abi_dict[field] for field in fields], [data[field] for field in fields], ) - formatted_dict = dict(zip(fields, formatted_values)) - return dict(data, **formatted_dict) + formatted_dict = data.copy() + formatted_dict.update(zip(fields, formatted_values)) + return formatted_dict @to_dict From 0ab501df038fa2acc3afae4aa6e88bf99f65e683 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 04/13] Create 3671.performance.rst --- newsfragments/3671.performance.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3671.performance.rst diff --git a/newsfragments/3671.performance.rst b/newsfragments/3671.performance.rst new file mode 100644 index 0000000000..13049a8c1e --- /dev/null +++ b/newsfragments/3671.performance.rst @@ -0,0 +1 @@ +optimize web3._utils.rpc_abi.apply_abi_formatters_to_dict From 9456a3cfaa944a866eca1fcd6f5593a46dc08c55 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 05/13] fix(types): update type hints in web3._utils.method_formatters.py --- web3/_utils/method_formatters.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py index 0136b6d30b..32c5923634 100644 --- a/web3/_utils/method_formatters.py +++ b/web3/_utils/method_formatters.py @@ -101,6 +101,7 @@ BlockIdentifier, Formatters, RPCEndpoint, + RPCResponse, SimulateV1Payload, StateOverrideParams, TReturn, @@ -600,19 +601,15 @@ def storage_key_to_hexstr(value: Union[bytes, int, str]) -> HexStr: ) block_result_formatters_copy = BLOCK_RESULT_FORMATTERS.copy() -block_result_formatters_copy.update( - { - "calls": apply_list_to_array_formatter( - type_aware_apply_formatters_to_dict( - { - "returnData": HexBytes, - "logs": apply_list_to_array_formatter(log_entry_formatter), - "gasUsed": to_integer_if_hex, - "status": to_integer_if_hex, - } - ) - ) - } +block_result_formatters_copy["calls"] = apply_list_to_array_formatter( + type_aware_apply_formatters_to_dict( + { + "returnData": HexBytes, + "logs": apply_list_to_array_formatter(log_entry_formatter), + "gasUsed": to_integer_if_hex, + "status": to_integer_if_hex, + } + ) ) simulate_v1_result_formatter = apply_formatter_if( is_not_null, @@ -1201,7 +1198,7 @@ def apply_module_to_formatters( def get_result_formatters( method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]], module: "Module", -) -> Dict[str, Callable[..., Any]]: +) -> Callable[[RPCResponse], Any]: formatters = combine_formatters((PYTHONIC_RESULT_FORMATTERS,), method_name) formatters_requiring_module = combine_formatters( (FILTER_RESULT_FORMATTERS,), method_name @@ -1214,7 +1211,7 @@ def get_result_formatters( def get_error_formatters( method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]] -) -> Callable[..., Any]: +) -> Callable[[RPCResponse], Any]: # Note error formatters work on the full response dict error_formatter_maps = (ERROR_FORMATTERS,) formatters = combine_formatters(error_formatter_maps, method_name) @@ -1224,7 +1221,7 @@ def get_error_formatters( def get_null_result_formatters( method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]] -) -> Callable[..., Any]: +) -> Callable[[RPCResponse], Any]: formatters = combine_formatters((NULL_RESULT_FORMATTERS,), method_name) return compose(*formatters) From 0412b8f69e483943956f9766e4878e299ceeade2 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 06/13] fix: Tuple -> Iterable --- web3/_utils/method_formatters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py index 32c5923634..e57254ec25 100644 --- a/web3/_utils/method_formatters.py +++ b/web3/_utils/method_formatters.py @@ -1187,7 +1187,7 @@ def filter_wrapper( @to_tuple def apply_module_to_formatters( - formatters: Tuple[Callable[..., TReturn]], + formatters: Iterable[Callable[..., TReturn]], module: "Module", method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]], ) -> Iterable[Callable[..., TReturn]]: From 343af490ac5fbd37b147840c822bd418c2b48fc2 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 07/13] feat: optimize reject_recursive_repeats --- web3/_utils/decorators.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/web3/_utils/decorators.py b/web3/_utils/decorators.py index 346ac5b045..7115ff7a7b 100644 --- a/web3/_utils/decorators.py +++ b/web3/_utils/decorators.py @@ -3,6 +3,8 @@ from typing import ( Any, Callable, + Set, + Tuple, TypeVar, cast, ) @@ -20,21 +22,22 @@ def reject_recursive_repeats(to_wrap: Callable[..., Any]) -> Callable[..., Any]: Prevent simple cycles by returning None when called recursively with same instance """ # types ignored b/c dynamically set attribute - to_wrap.__already_called = {} # type: ignore + already_called: Set[Tuple[int, ...]] = set() + to_wrap.__already_called = already_called # type: ignore + + add_call = already_called.add + remove_call = already_called.remove @functools.wraps(to_wrap) def wrapped(*args: Any) -> Any: - arg_instances = tuple(map(id, args)) - thread_id = threading.get_ident() - thread_local_args = (thread_id,) + arg_instances - if thread_local_args in to_wrap.__already_called: # type: ignore + thread_local_args = (threading.get_ident(), *map(id, args)) + if thread_local_args in already_called: raise Web3ValueError(f"Recursively called {to_wrap} with {args!r}") - to_wrap.__already_called[thread_local_args] = True # type: ignore + add_call(thread_local_args) try: - wrapped_val = to_wrap(*args) + return to_wrap(*args) finally: - del to_wrap.__already_called[thread_local_args] # type: ignore - return wrapped_val + remove_call(thread_local_args) return wrapped From e22506ec67ff3206b8885278651ebb60955f6f70 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 08/13] chore: add newsfragment --- newsfragments/3668.performance.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3668.performance.rst diff --git a/newsfragments/3668.performance.rst b/newsfragments/3668.performance.rst new file mode 100644 index 0000000000..f002a0505a --- /dev/null +++ b/newsfragments/3668.performance.rst @@ -0,0 +1 @@ +Optimize web3._utils.decorators.reject_recursive_repeats From 129c18396dabc4ef83faa0ab50e8b639d4fee7a1 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 09/13] feat: optimize utility_methods.py This was originally part of my other PR but that is becoming quite cluttered and hard to review so I separated this one out. Previously, the code would create a new dict from `d` one time for each item in the iterable. Now, the code creates just one new dict from `d` at the beginning and uses that same dict to check membership for each item. --- web3/_utils/utility_methods.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/web3/_utils/utility_methods.py b/web3/_utils/utility_methods.py index 66ad373ff2..45051655d8 100644 --- a/web3/_utils/utility_methods.py +++ b/web3/_utils/utility_methods.py @@ -1,7 +1,7 @@ from typing import ( Any, - Dict, Iterable, + Mapping, Set, Union, ) @@ -13,7 +13,7 @@ def all_in_dict( - values: Iterable[Any], d: Union[Dict[Any, Any], TxData, TxParams] + values: Iterable[Any], d: Union[Mapping[Any, Any], TxData, TxParams] ) -> bool: """ Returns a bool based on whether ALL of the provided values exist @@ -24,11 +24,12 @@ def all_in_dict( :return: True if ALL values exist in keys; False if NOT ALL values exist in keys """ - return all(_ in dict(d) for _ in values) + d = dict(d) + return all(_ in d for _ in values) def any_in_dict( - values: Iterable[Any], d: Union[Dict[Any, Any], TxData, TxParams] + values: Iterable[Any], d: Union[Mapping[Any, Any], TxData, TxParams] ) -> bool: """ Returns a bool based on whether ANY of the provided values exist @@ -39,11 +40,12 @@ def any_in_dict( :return: True if ANY value exists in keys; False if NONE of the values exist in keys """ - return any(_ in dict(d) for _ in values) + d = dict(d) + return any(_ in d for _ in values) def none_in_dict( - values: Iterable[Any], d: Union[Dict[Any, Any], TxData, TxParams] + values: Iterable[Any], d: Union[Mapping[Any, Any], TxData, TxParams] ) -> bool: """ Returns a bool based on whether NONE of the provided values exist From 3e95a9a6d45465b4cb69f89239c106e4911cb57c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 18 Apr 2025 20:39:31 +0000 Subject: [PATCH 10/13] chore: add newsfragment --- newsfragments/3667.performance.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 newsfragments/3667.performance.rst diff --git a/newsfragments/3667.performance.rst b/newsfragments/3667.performance.rst new file mode 100644 index 0000000000..eb582f6f1c --- /dev/null +++ b/newsfragments/3667.performance.rst @@ -0,0 +1,4 @@ +Optimize performance for: +web3._utils.utility_methods.all_in_dict +web3._utils.utility_methods.any_in_dict +web3._utils.utility_methods.none_in_dict From f42e75ff99f0d94fff4af3cbf1abe1f1b7d049e9 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Fri, 18 Apr 2025 20:39:45 +0000 Subject: [PATCH 11/13] feat: cache error and null result formatters --- web3/_utils/method_formatters.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py index e57254ec25..dbd10ecb07 100644 --- a/web3/_utils/method_formatters.py +++ b/web3/_utils/method_formatters.py @@ -116,6 +116,11 @@ TValue = TypeVar("TValue") +CachedFormatters = Dict[ + Union[RPCEndpoint, Callable[..., RPCEndpoint]], + Callable[[RPCResponse], Any], +] + def bytes_to_ascii(value: bytes) -> str: return codecs.decode(value, "ascii") @@ -1209,19 +1214,28 @@ def get_result_formatters( return compose(*partial_formatters, *formatters) +_error_formatters: CachedFormatters = {} + def get_error_formatters( method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]] ) -> Callable[[RPCResponse], Any]: # Note error formatters work on the full response dict - error_formatter_maps = (ERROR_FORMATTERS,) - formatters = combine_formatters(error_formatter_maps, method_name) + formatters = _error_formatters.get(method_name) + if formatters is None: + formatters = _error_formatters[method_name] = compose( + *combine_formatters((ERROR_FORMATTERS,), method_name) + ) + return formatters - return compose(*formatters) +_null_result_formatters: CachedFormatters = {} def get_null_result_formatters( method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]] ) -> Callable[[RPCResponse], Any]: - formatters = combine_formatters((NULL_RESULT_FORMATTERS,), method_name) - - return compose(*formatters) + formatters = _null_result_formatters.get(method_name) + if formatters is None: + formatters = _null_result_formatters[method_name] = compose( + *combine_formatters((NULL_RESULT_FORMATTERS,), method_name) + ) + return formatters From 67b1fbc85dd093bcb316fd0d34d4f3ed571a5506 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Fri, 18 Apr 2025 20:39:54 +0000 Subject: [PATCH 12/13] feat: optimize with iterators --- web3/_utils/method_formatters.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py index dbd10ecb07..2a69a22bd9 100644 --- a/web3/_utils/method_formatters.py +++ b/web3/_utils/method_formatters.py @@ -7,6 +7,7 @@ Collection, Dict, Iterable, + Iterator, NoReturn, Tuple, TypeVar, @@ -35,7 +36,6 @@ is_string, to_checksum_address, to_list, - to_tuple, ) from eth_utils.toolz import ( complement, @@ -1048,11 +1048,10 @@ def subscription_formatter(value: Any) -> Union[HexBytes, HexStr, Dict[str, Any] } -@to_tuple def combine_formatters( formatter_maps: Collection[Dict[RPCEndpoint, Callable[..., TReturn]]], method_name: RPCEndpoint, -) -> Iterable[Callable[..., TReturn]]: +) -> Iterator[Callable[..., TReturn]]: for formatter_map in formatter_maps: if method_name in formatter_map: yield formatter_map[method_name] @@ -1190,12 +1189,11 @@ def filter_wrapper( } -@to_tuple def apply_module_to_formatters( formatters: Iterable[Callable[..., TReturn]], module: "Module", method_name: Union[RPCEndpoint, Callable[..., RPCEndpoint]], -) -> Iterable[Callable[..., TReturn]]: +) -> Iterator[Callable[..., TReturn]]: for f in formatters: yield partial(f, module, method_name) From 10f8fb8e21f4bd9d7d186e08e3532bc6c768a035 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 9 May 2025 15:52:36 -0400 Subject: [PATCH 13/13] Delete newsfragments/3667.performance.rst already merged to master --- newsfragments/3667.performance.rst | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 newsfragments/3667.performance.rst diff --git a/newsfragments/3667.performance.rst b/newsfragments/3667.performance.rst deleted file mode 100644 index eb582f6f1c..0000000000 --- a/newsfragments/3667.performance.rst +++ /dev/null @@ -1,4 +0,0 @@ -Optimize performance for: -web3._utils.utility_methods.all_in_dict -web3._utils.utility_methods.any_in_dict -web3._utils.utility_methods.none_in_dict