From 2eea05b7edf420309bff0685f0d1e59e96d5dc7c Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 26 Apr 2023 15:46:12 -0600 Subject: [PATCH 1/5] On older versions of Python, skip benchmarks that use features introduced in newer Python versions --- pyperformance/_benchmark.py | 7 ++++++- pyperformance/cli.py | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pyperformance/_benchmark.py b/pyperformance/_benchmark.py index 398b2049..484d59e1 100644 --- a/pyperformance/_benchmark.py +++ b/pyperformance/_benchmark.py @@ -13,6 +13,7 @@ import sys import pyperf +from packaging.specifiers import SpecifierSet from . import _utils, _benchmark_metadata @@ -164,9 +165,13 @@ def runscript(self): def extra_opts(self): return self._get_metadata_value('extra_opts', ()) + @property + def python(self): + req = self._get_metadata_value("python", None) + return None if req is None else SpecifierSet(req) + # Other metadata keys: # * base - # * python # * dependencies # * requirements diff --git a/pyperformance/cli.py b/pyperformance/cli.py index 4fe3ca1a..a0d46b17 100644 --- a/pyperformance/cli.py +++ b/pyperformance/cli.py @@ -241,11 +241,15 @@ def parse_entry(o, s): # Get the selections. selected = [] + this_python_version = ".".join(map(str, sys.version_info[:3])) for bench in _benchmark_selections.iter_selections(manifest, parsed_infos): if isinstance(bench, str): logging.warning(f"no benchmark named {bench!r}") continue - selected.append(bench) + # Filter out any benchmarks that can't be run on the Python version we're running + if bench.python is not None and this_python_version in bench.python: + selected.append(bench) + return selected From 8153afb76ce8624ae6f56984bab40065f15bdb0a Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 26 Apr 2023 16:04:59 -0600 Subject: [PATCH 2/5] Fix tests on 3.7 --- pyperformance/tests/data/bm_local_wheel/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyperformance/tests/data/bm_local_wheel/pyproject.toml b/pyperformance/tests/data/bm_local_wheel/pyproject.toml index a3191642..2710ced7 100644 --- a/pyperformance/tests/data/bm_local_wheel/pyproject.toml +++ b/pyperformance/tests/data/bm_local_wheel/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pyperformance_bm_local_wheel" -requires-python = ">=3.8" +requires-python = ">=3.7" dependencies = ["pyperf"] urls = {repository = "https://github.com/python/pyperformance"} version = "1.0" From 1734919c7ca476aba40b03121e310467878a2702 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 26 Apr 2023 16:20:41 -0600 Subject: [PATCH 3/5] Add a benchmark that uses features new in 3.8 --- pyperformance/data-files/benchmarks/MANIFEST | 1 + .../pyproject.toml | 9 + .../run_benchmark.py | 180 ++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml create mode 100644 pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index d472c2c1..01c87603 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -69,6 +69,7 @@ sympy telco tomli_loads tornado_http +typing_runtime_protocols unpack_sequence unpickle unpickle_list diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml new file mode 100644 index 00000000..a69cf598 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "pyperformance_bm_typing_runtime_protocols" +requires-python = ">=3.8" +dependencies = ["pyperf"] +urls = {repository = "https://github.com/python/pyperformance"} +dynamic = ["version"] + +[tool.pyperformance] +name = "typing_runtime_protocols" diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py new file mode 100644 index 00000000..dd8e7ef3 --- /dev/null +++ b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py @@ -0,0 +1,180 @@ +""" +Test the performance of isinstance() checks against runtime-checkable protocols. + +For programmes that make extensive use of this feature, +these calls can easily become a bottleneck. +See https://github.com/python/cpython/issues/74690 + +The following situations all exercise different code paths +in typing._ProtocolMeta.__instancecheck__, +so each is tested in this benchmark: + + (1) Comparing an instance of a class that directly inherits + from a protocol to that protocol. + (2) Comparing an instance of a class that fulfils the interface + of a protocol using instance attributes. + (3) Comparing an instance of a class that fulfils the interface + of a protocol using class attributes. + (4) Comparing an instance of a class that fulfils the interface + of a protocol using properties. + +Protocols with callable and non-callable members also +exercise different code paths in _ProtocolMeta.__instancecheck__, +so are also tested separately. +""" + +import time +from typing import Protocol, runtime_checkable + +import pyperf + + +################################################## +# Protocols to call isinstance() against +################################################## + + +@runtime_checkable +class HasX(Protocol): + """A runtime-checkable protocol with a single non-callable member""" + x: int + +@runtime_checkable +class HasManyAttributes(Protocol): + """A runtime-checkable protocol with many non-callable members""" + a: int + b: int + c: int + d: int + e: int + +@runtime_checkable +class SupportsInt(Protocol): + """A runtime-checkable protocol with a single callable member""" + def __int__(self) -> int: ... + +@runtime_checkable +class SupportsManyMethods(Protocol): + """A runtime-checkable protocol with many callable members""" + def one(self) -> int: ... + def two(self) -> str: ... + def three(self) -> bytes: ... + def four(self) -> memoryview: ... + def five(self) -> bytearray: ... + +@runtime_checkable +class SupportsIntAndX(Protocol): + """A runtime-checkable protocol with a mix of callable and non-callable members""" + x: int + def __int__(self) -> int: ... + + +################################################## +# Classes for comparing against the protocols +################################################## + + +class Empty: + """Empty class with no attributes""" + +class PropertyX: + """Class with a property x""" + @property + def x(self) -> int: return 42 + +class HasIntMethod: + """Class with an __int__ method""" + def __int__(self): return 42 + +class HasManyMethods: + """Class with many methods""" + def one(self): return 42 + def two(self): return "42" + def three(self): return b"42" + def four(self): return memoryview(b"42") + def five(self): return bytearray(b"42") + +class PropertyXWithInt: + """Class with a property x and an __int__ method""" + @property + def x(self) -> int: return 42 + def __int__(self): return 42 + +class ClassVarX: + """Class with a ClassVar x""" + x = 42 + +class ClassVarXWithInt: + """Class with a ClassVar x and an __int__ method""" + x = 42 + def __int__(self): return 42 + +class InstanceVarX: + """Class with an instance var x""" + def __init__(self): + self.x = 42 + +class ManyInstanceVars: + """Class with many instance vars""" + def __init__(self): + for attr in 'abcde': + setattr(self, attr, 42) + +class InstanceVarXWithInt: + """Class with an instance var x and an __int__ method""" + def __init__(self): + self.x = 42 + def __int__(self): + return 42 + +class NominalX(HasX): + """Class that explicitly subclasses HasX""" + def __init__(self): + self.x = 42 + +class NominalSupportsInt(SupportsInt): + """Class that explicitly subclasses SupportsInt""" + def __int__(self): + return 42 + +class NominalXWithInt(SupportsIntAndX): + """Class that explicitly subclasses NominalXWithInt""" + def __init__(self): + self.x = 42 + + +################################################## +# Time to test the performance of isinstance()! +################################################## + + +def bench_protocols(loops: int) -> float: + protocols = [ + HasX, HasManyAttributes, SupportsInt, SupportsManyMethods, SupportsIntAndX + ] + instances = [ + cls() + for cls in ( + Empty, PropertyX, HasIntMethod, HasManyMethods, PropertyXWithInt, + ClassVarX, ClassVarXWithInt, InstanceVarX, ManyInstanceVars, + InstanceVarXWithInt, NominalX, NominalSupportsInt, NominalXWithInt + ) + ] + + t0 = time.perf_counter() + + for _ in range(loops): + for protocol in protocols: + for instance in instances: + isinstance(instance, protocol) + + return time.perf_counter() - t0 + + +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata["description"] = ( + "Test the performance of isinstance() checks " + "against runtime-checkable protocols" + ) + runner.bench_time_func("typing_runtime_protocols", bench_protocols) From 65e11619a8aad28eba2aeabee71e959b9b33a902 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 26 Apr 2023 16:24:48 -0600 Subject: [PATCH 4/5] Revert "Add a benchmark that uses features new in 3.8" This reverts commit 1734919c7ca476aba40b03121e310467878a2702. --- pyperformance/data-files/benchmarks/MANIFEST | 1 - .../pyproject.toml | 9 - .../run_benchmark.py | 180 ------------------ 3 files changed, 190 deletions(-) delete mode 100644 pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml delete mode 100644 pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py diff --git a/pyperformance/data-files/benchmarks/MANIFEST b/pyperformance/data-files/benchmarks/MANIFEST index 01c87603..d472c2c1 100644 --- a/pyperformance/data-files/benchmarks/MANIFEST +++ b/pyperformance/data-files/benchmarks/MANIFEST @@ -69,7 +69,6 @@ sympy telco tomli_loads tornado_http -typing_runtime_protocols unpack_sequence unpickle unpickle_list diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml deleted file mode 100644 index a69cf598..00000000 --- a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[project] -name = "pyperformance_bm_typing_runtime_protocols" -requires-python = ">=3.8" -dependencies = ["pyperf"] -urls = {repository = "https://github.com/python/pyperformance"} -dynamic = ["version"] - -[tool.pyperformance] -name = "typing_runtime_protocols" diff --git a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py b/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py deleted file mode 100644 index dd8e7ef3..00000000 --- a/pyperformance/data-files/benchmarks/bm_typing_runtime_protocols/run_benchmark.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -Test the performance of isinstance() checks against runtime-checkable protocols. - -For programmes that make extensive use of this feature, -these calls can easily become a bottleneck. -See https://github.com/python/cpython/issues/74690 - -The following situations all exercise different code paths -in typing._ProtocolMeta.__instancecheck__, -so each is tested in this benchmark: - - (1) Comparing an instance of a class that directly inherits - from a protocol to that protocol. - (2) Comparing an instance of a class that fulfils the interface - of a protocol using instance attributes. - (3) Comparing an instance of a class that fulfils the interface - of a protocol using class attributes. - (4) Comparing an instance of a class that fulfils the interface - of a protocol using properties. - -Protocols with callable and non-callable members also -exercise different code paths in _ProtocolMeta.__instancecheck__, -so are also tested separately. -""" - -import time -from typing import Protocol, runtime_checkable - -import pyperf - - -################################################## -# Protocols to call isinstance() against -################################################## - - -@runtime_checkable -class HasX(Protocol): - """A runtime-checkable protocol with a single non-callable member""" - x: int - -@runtime_checkable -class HasManyAttributes(Protocol): - """A runtime-checkable protocol with many non-callable members""" - a: int - b: int - c: int - d: int - e: int - -@runtime_checkable -class SupportsInt(Protocol): - """A runtime-checkable protocol with a single callable member""" - def __int__(self) -> int: ... - -@runtime_checkable -class SupportsManyMethods(Protocol): - """A runtime-checkable protocol with many callable members""" - def one(self) -> int: ... - def two(self) -> str: ... - def three(self) -> bytes: ... - def four(self) -> memoryview: ... - def five(self) -> bytearray: ... - -@runtime_checkable -class SupportsIntAndX(Protocol): - """A runtime-checkable protocol with a mix of callable and non-callable members""" - x: int - def __int__(self) -> int: ... - - -################################################## -# Classes for comparing against the protocols -################################################## - - -class Empty: - """Empty class with no attributes""" - -class PropertyX: - """Class with a property x""" - @property - def x(self) -> int: return 42 - -class HasIntMethod: - """Class with an __int__ method""" - def __int__(self): return 42 - -class HasManyMethods: - """Class with many methods""" - def one(self): return 42 - def two(self): return "42" - def three(self): return b"42" - def four(self): return memoryview(b"42") - def five(self): return bytearray(b"42") - -class PropertyXWithInt: - """Class with a property x and an __int__ method""" - @property - def x(self) -> int: return 42 - def __int__(self): return 42 - -class ClassVarX: - """Class with a ClassVar x""" - x = 42 - -class ClassVarXWithInt: - """Class with a ClassVar x and an __int__ method""" - x = 42 - def __int__(self): return 42 - -class InstanceVarX: - """Class with an instance var x""" - def __init__(self): - self.x = 42 - -class ManyInstanceVars: - """Class with many instance vars""" - def __init__(self): - for attr in 'abcde': - setattr(self, attr, 42) - -class InstanceVarXWithInt: - """Class with an instance var x and an __int__ method""" - def __init__(self): - self.x = 42 - def __int__(self): - return 42 - -class NominalX(HasX): - """Class that explicitly subclasses HasX""" - def __init__(self): - self.x = 42 - -class NominalSupportsInt(SupportsInt): - """Class that explicitly subclasses SupportsInt""" - def __int__(self): - return 42 - -class NominalXWithInt(SupportsIntAndX): - """Class that explicitly subclasses NominalXWithInt""" - def __init__(self): - self.x = 42 - - -################################################## -# Time to test the performance of isinstance()! -################################################## - - -def bench_protocols(loops: int) -> float: - protocols = [ - HasX, HasManyAttributes, SupportsInt, SupportsManyMethods, SupportsIntAndX - ] - instances = [ - cls() - for cls in ( - Empty, PropertyX, HasIntMethod, HasManyMethods, PropertyXWithInt, - ClassVarX, ClassVarXWithInt, InstanceVarX, ManyInstanceVars, - InstanceVarXWithInt, NominalX, NominalSupportsInt, NominalXWithInt - ) - ] - - t0 = time.perf_counter() - - for _ in range(loops): - for protocol in protocols: - for instance in instances: - isinstance(instance, protocol) - - return time.perf_counter() - t0 - - -if __name__ == "__main__": - runner = pyperf.Runner() - runner.metadata["description"] = ( - "Test the performance of isinstance() checks " - "against runtime-checkable protocols" - ) - runner.bench_time_func("typing_runtime_protocols", bench_protocols) From a27d70888c116ea64e13ae45142cf4240641b9b3 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 26 Apr 2023 20:08:54 -0600 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Brandt Bucher --- pyperformance/_benchmark.py | 3 +-- pyperformance/cli.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyperformance/_benchmark.py b/pyperformance/_benchmark.py index 484d59e1..0cf8f2c7 100644 --- a/pyperformance/_benchmark.py +++ b/pyperformance/_benchmark.py @@ -167,8 +167,7 @@ def extra_opts(self): @property def python(self): - req = self._get_metadata_value("python", None) - return None if req is None else SpecifierSet(req) + return SpecifierSet(self._get_metadata_value("python", "")) # Other metadata keys: # * base diff --git a/pyperformance/cli.py b/pyperformance/cli.py index a0d46b17..330d6e57 100644 --- a/pyperformance/cli.py +++ b/pyperformance/cli.py @@ -247,7 +247,7 @@ def parse_entry(o, s): logging.warning(f"no benchmark named {bench!r}") continue # Filter out any benchmarks that can't be run on the Python version we're running - if bench.python is not None and this_python_version in bench.python: + if this_python_version in bench.python: selected.append(bench) return selected