From db1e16748de02beab3d27c82bad66e0bfc016ea6 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Fri, 14 Mar 2025 18:27:20 +0000 Subject: [PATCH 1/3] Tidy up infra --- .github/workflows/test.yml | 2 +- pyproject.toml | 36 ++++++++++++++++++------------------ src/vault_dev/__init__.py | 2 +- src/vault_dev/py.typed | 0 4 files changed, 20 insertions(+), 20 deletions(-) create mode 100644 src/vault_dev/py.typed diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e2f173..b176829 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: hatch run cov-ci - name: Lint run: | - hatch run lint:style + hatch run lint:all - name: Upload to Codecov uses: codecov/codecov-action@v3 with: diff --git a/pyproject.toml b/pyproject.toml index 7f0c5aa..09b6fbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,9 @@ build-backend = "hatchling.build" [project] name = "vault-dev" -dynamic = ["version"] -description = '' +description = "Test helpers for using vault" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.10" license = "MIT" keywords = [] authors = [ @@ -16,11 +15,10 @@ authors = [ classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] @@ -29,6 +27,7 @@ dependencies = [ "hvac", "requests" ] +dynamic = ["version"] [project.urls] Documentation = "https://github.com/vimc/vault-dev#readme" @@ -40,8 +39,10 @@ path = "src/vault_dev/__about__.py" [tool.hatch.envs.default] dependencies = [ - "coverage[toml]>=6.5", "pytest", + "pytest-cov", + "pytest-mock", + "coverage[toml]>=6.5", ] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" @@ -64,24 +65,23 @@ cov-ci = [ ] [[tool.hatch.envs.all.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11"] +python = ["3.10", "3.11", "3.12", "3.13"] [tool.hatch.envs.lint] -detached = true -dependencies = [ +extra-dependencies = [ "black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243", ] [tool.hatch.envs.lint.scripts] -typing = "mypy --install-types --non-interactive {args:src/vault_dev tests}" +typing = "mypy --install-types --non-interactive {args:src tests}" style = [ - "ruff {args:.}", + "ruff check {args:.}", "black --check --diff {args:.}", ] fmt = [ "black {args:.}", - "ruff --fix {args:.}", + "ruff check --fix {args:.}", "style", ] all = [ @@ -90,13 +90,13 @@ all = [ ] [tool.black] -target-version = ["py37"] line-length = 80 skip-string-normalization = true [tool.ruff] -target-version = "py37" line-length = 80 + +[tool.ruff.lint] select = [ "A", "ARG", @@ -143,13 +143,13 @@ unfixable = [ "F401", ] -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["vault_dev"] -[tool.ruff.flake8-tidy-imports] +[tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] diff --git a/src/vault_dev/__init__.py b/src/vault_dev/__init__.py index 869b2d7..a9adbe3 100644 --- a/src/vault_dev/__init__.py +++ b/src/vault_dev/__init__.py @@ -9,4 +9,4 @@ class server(Server): # noqa N801 pass -__all__ = ["Server", "VaultDevServerError", "server", "ensure_installed"] +__all__ = ["Server", "VaultDevServerError", "ensure_installed", "server"] diff --git a/src/vault_dev/py.typed b/src/vault_dev/py.typed new file mode 100644 index 0000000..e69de29 From 92265138e89efc924f2d438c2485fe821d5919d6 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Fri, 14 Mar 2025 18:45:06 +0000 Subject: [PATCH 2/3] Add types --- src/vault_dev/install.py | 26 +++++++++++++------------- src/vault_dev/server.py | 38 ++++++++++++++++++++++---------------- src/vault_dev/utils.py | 32 ++++++++++++++------------------ 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/vault_dev/install.py b/src/vault_dev/install.py index 5bd9394..c212f8d 100644 --- a/src/vault_dev/install.py +++ b/src/vault_dev/install.py @@ -18,36 +18,36 @@ import requests -def ensure_installed(): +def ensure_installed() -> None: if not shutil.which("vault"): print("Did not find system vault, installing one for tests") global_vault_dev_exe.install() -def vault_path(): +def vault_path() -> str: vault = shutil.which("vault") if not vault: vault = global_vault_dev_exe.vault() return vault -def vault_exe_filename(platform): +def vault_exe_filename(platform: str) -> str: if platform == "windows": return "vault.exe" else: return "vault" -def vault_url(version, platform, arch="amd64"): - fmt = "https://releases.hashicorp.com/vault/{}/vault_{}_{}_{}.zip" - return fmt.format(version, version, platform, arch) +def vault_url(version: str, platform: str, arch: str = "amd64") -> str: + base = "https://releases.hashicorp.com/vault" + return f"{base}/{version}/vault_{version}_{platform}_{arch}.zip" -def vault_platform(): +def vault_platform() -> str: return platform.system().lower() -def vault_download(dest, version, platform): +def vault_download(dest: str, version: str, platform: str) -> str: dest_bin = f"{dest}/{vault_exe_filename(platform)}" if not os.path.exists(dest_bin): print(f"installing vault to '{dest}'") @@ -63,8 +63,8 @@ def vault_download(dest, version, platform): class VaultDevExe: - path = None - exe = None + path: str | None = None + exe: str | None = None def __init__(self): if not self.path: @@ -73,13 +73,13 @@ def __init__(self): self.path = tmp self.exe = f"{tmp.name}/{exe}" - def install(self): + def install(self) -> str: return vault_download(self.path.name, "1.0.0", vault_platform()) - def exists(self): + def exists(self) -> bool: return self.exe and os.path.exists(self.exe) - def vault(self): + def vault(self) -> str: if self.exists(): return self.exe else: diff --git a/src/vault_dev/server.py b/src/vault_dev/server.py index b9a4ac7..f6cc896 100644 --- a/src/vault_dev/server.py +++ b/src/vault_dev/server.py @@ -7,23 +7,29 @@ import hvac from vault_dev.install import vault_path -from vault_dev.utils import find_free_port, read_all_lines, transient_envvar +from vault_dev.utils import find_free_port, read_all_lines, transient_envvars class Server: + process: subprocess.Popen | None = None + _prev_token: str | None = None + def __init__( - self, *, port=None, verbose=False, debug=False, export_token=False + self, + *, + port: int | None = None, + verbose: bool = False, + debug: bool = False, + export_token: bool = False, ): - self.process = None self.verbose = verbose or debug self.vault = vault_path() self.port = port or find_free_port() self.token = str(uuid.uuid4()) self.debug = debug self.export_token = export_token - self._prev_token = None - def start(self, timeout=5, poll=0.1): + def start(self, timeout: float = 5, poll: float = 0.1) -> None: if self.is_running(): self._message("Vault server already started") return @@ -46,7 +52,7 @@ def start(self, timeout=5, poll=0.1): self._prev_token = os.environ["VAULT_TOKEN"] os.environ["VAULT_TOKEN"] = self.token - def stop(self, *, wait=True): + def stop(self, *, wait: bool = True) -> None: self._message("Stopping vault server") if self.export_token: if self._prev_token is None: @@ -54,35 +60,35 @@ def stop(self, *, wait=True): del os.environ["VAULT_TOKEN"] else: os.environ["VAULT_TOKEN"] = self._prev_token - if self.is_running(): + if self.process and not self.process.poll(): self.process.kill() if wait: self.process.wait() - def client(self): + def client(self) -> hvac.Client: # See https://github.com/hvac/hvac/issues/421 - with transient_envvar(VAULT_ADDR=None, VAULT_TOKEN=None): + with transient_envvars({"VAULT_ADDR": None, "VAULT_TOKEN": None}): client = hvac.Client(url=self.url(), token=self.token) assert client.is_authenticated() # noqa S101 return client - def url(self): + def url(self) -> str: return f"http://localhost:{self.port}" - def __enter__(self): + def __enter__(self) -> "Server": self.start() return self - def __exit__(self, ex_type, ex_value, traceback): + def __exit__(self, ex_type, ex_value, traceback) -> None: self.stop() - def __del__(self): + def __del__(self) -> None: self.stop(wait=False) def is_running(self): return self.process and not self.process.poll() - def _wait_until_active(self, timeout, poll): + def _wait_until_active(self, timeout: float, poll: float) -> None: self._message("Waiting for server to become active") for _i in range(math.ceil(timeout / poll)): if not self.is_running(): @@ -99,7 +105,7 @@ def _wait_until_active(self, timeout, poll): msg = "Vault did not start in time" raise VaultDevServerError(msg, self.process) - def _enable_kv1(self): + def _enable_kv1(self) -> None: self._message("Configuring old-style kv engine at /secret") cl = self.client() cl.sys.disable_secrets_engine(path="secret") @@ -107,7 +113,7 @@ def _enable_kv1(self): backend_type="kv", path="secret", options={"version": 1} ) - def _message(self, txt, **kwargs): + def _message(self, txt: str, **kwargs) -> None: if self.verbose: print(txt, **kwargs) diff --git a/src/vault_dev/utils.py b/src/vault_dev/utils.py index eab8018..8f75770 100644 --- a/src/vault_dev/utils.py +++ b/src/vault_dev/utils.py @@ -1,9 +1,10 @@ import os import socket +from collections.abc import Iterator from contextlib import contextmanager -def find_free_port(): +def find_free_port() -> int: with socket.socket() as s: # Let the OS pick a random free port on our machine s.bind(("localhost", 0)) @@ -15,7 +16,7 @@ def find_free_port(): return s.getsockname()[1] -def read_all_lines(con, prefix=""): +def read_all_lines(con, prefix="") -> str: output = [] while True: d = con.readline() @@ -26,22 +27,17 @@ def read_all_lines(con, prefix=""): @contextmanager -def transient_envvar(**kwargs): - prev = { - k: os.environ[k] if k in os.environ else None for k in kwargs.keys() - } +def transient_envvars(env: dict[str, str | None]) -> Iterator[None]: + def _set_envvars(env): + for k, v in env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + prev = {k: os.environ.get(k) for k in env.keys()} try: - _setdictvals(kwargs, os.environ) + _set_envvars(env) yield finally: - _setdictvals(prev, os.environ) - - -def _setdictvals(new, container): - for k, v in new.items(): - if v is None: - if k in container: - del container[k] - else: - container[k] = v - return container + _set_envvars(prev) From 8c0e01631a97782823ced45b5a7a9ecb307ed46c Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Mon, 17 Mar 2025 07:31:59 +0000 Subject: [PATCH 3/3] Simplify handling of installation --- pyproject.toml | 12 +++ src/vault_dev/install.py | 169 ++++++++++++++++++++++++++------------- src/vault_dev/server.py | 2 +- tests/test_install.py | 49 ++++++------ 4 files changed, 148 insertions(+), 84 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 09b6fbd..e5f0e48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ path = "src/vault_dev/__about__.py" dependencies = [ "pytest", "pytest-cov", + "pytest-explicit", "pytest-mock", "coverage[toml]>=6.5", ] @@ -171,3 +172,14 @@ exclude_lines = [ "if __name__ == .__main__.:", "if TYPE_CHECKING:", ] + + +[tool.pytest.ini_options] +markers = [ + "slow: slower integration tests", + "internet: tests that require internet", +] +explicit-only = [ + "internet", + "slow", +] diff --git a/src/vault_dev/install.py b/src/vault_dev/install.py index c212f8d..7bb2fc2 100644 --- a/src/vault_dev/install.py +++ b/src/vault_dev/install.py @@ -1,92 +1,147 @@ -## This whole file struggles with the problem of installing a global -## resource that is somewhat expensive (a vault binary, needed for -## installing tests) and we want to just use a system binary if it's -## already found. -## -## The only entrypoint that other packages need to be concerned with -## is "ensure_installed()" which can be run with no arguments and -## ensures that a suitable vault binary is installed. -## -## This package then uses the "vault_path()" function to get the path -## to either the system vault or the one that was installed. +"""Support for finding and installing the vault binary. + +Use of this package requires that you have vault installed and +available. It is only a single binary, so not very hard to install, +but the package will be used in situations where we don't want to +think about this too much. + +If the environment variable `$VAULT_DEV_BIN_PATH` is set, then this is +preferred (and if vault is not found here an error is thrown). This +can be the full path to the vault binary (include `.exe` on windows) +or to the directory in which `vault` or `vault.exe` is found. + +If vault is present on `$PATH` we will use that. + +We provide some support for downloading and installing vault, by +passing `install=True` to `ensure_installed()`. + +If using GitHub actions, you can use an action +(e.g. `eLco/setup-vault`) to do this. + +""" + import os import platform import shutil import tempfile import zipfile +from pathlib import Path import requests -def ensure_installed() -> None: - if not shutil.which("vault"): - print("Did not find system vault, installing one for tests") - global_vault_dev_exe.install() +def vault_path(*, required: bool = True) -> Path | None: + """Compute path to vault binary + Args: + required: Throw if vault is not found -def vault_path() -> str: - vault = shutil.which("vault") - if not vault: - vault = global_vault_dev_exe.vault() - return vault + Returns: The path to the vault binary, or `None` if not found, and + if `required` is `False`. + """ + if path := _find_vault_environ(): + return path + if path := _find_vault_system(): + return path + if required: + msg = "vault not found" + raise Exception(msg) + return None -def vault_exe_filename(platform: str) -> str: - if platform == "windows": - return "vault.exe" - else: - return "vault" +def ensure_installed(*, install: bool = False) -> None: + """Ensure that vault is installed. -def vault_url(version: str, platform: str, arch: str = "amd64") -> str: - base = "https://releases.hashicorp.com/vault" - return f"{base}/{version}/vault_{version}_{platform}_{arch}.zip" + Args: + install: Install vault if not found? If not already installed + and if `install` is `False`, this function will throw. + Returns: + Nothing, called for side effects only. + + """ + path = vault_path(required=install) + if path: + print(f"Found vault at '{path}'") + if not path: + print("Did not find system vault, installing one for tests") + path = _vault_install_session() + print(f"Using vault at '{path}'") -def vault_platform() -> str: - return platform.system().lower() +def vault_download(dest: str | Path, version: str, platform: str) -> Path: + """Download vault. -def vault_download(dest: str, version: str, platform: str) -> str: - dest_bin = f"{dest}/{vault_exe_filename(platform)}" - if not os.path.exists(dest_bin): + Args: + dest: Destination directory + version: The version to download + platform: The platform to download (`linux`, `windows` or `darwin`) + + Returns: The full path to the downloaded binary. + """ + filename = _vault_exe_filename(platform) + dest_bin = Path(dest) / filename + if not dest_bin.exists(): print(f"installing vault to '{dest}'") - url = vault_url(version, platform) + url = _vault_url(version, platform) data = requests.get(url, timeout=600).content with tempfile.TemporaryFile() as tmp: tmp.write(data) tmp.seek(0) z = zipfile.ZipFile(tmp) - z.extract(vault_exe_filename(platform), dest) + z.extract(filename, dest) os.chmod(dest_bin, 0o755) # noqa S103 return dest_bin -class VaultDevExe: - path: str | None = None - exe: str | None = None +def _find_vault_environ() -> Path | None: + path = os.environ.get("VAULT_DEV_BIN_PATH") + if not path: + return None + ret = Path(path) + if not ret.exists(): + msg = "VAULT_DEV_BIN_PATH is set, but does not exist" + raise Exception(msg) + if ret.is_dir(): + filename = _vault_exe_filename(_vault_platform()) + ret = ret / filename + if not ret.exists(): + msg = f"{filename} not found at {ret}, from VAULT_DEV_BIN_PATH" + raise Exception(msg) + return ret - def __init__(self): - if not self.path: - tmp = tempfile.TemporaryDirectory() - exe = vault_exe_filename(vault_platform()) - self.path = tmp - self.exe = f"{tmp.name}/{exe}" - def install(self) -> str: - return vault_download(self.path.name, "1.0.0", vault_platform()) +def _find_vault_system() -> Path | None: + if path := shutil.which("vault"): + return Path(path) + else: + return None - def exists(self) -> bool: - return self.exe and os.path.exists(self.exe) - def vault(self) -> str: - if self.exists(): - return self.exe - else: - msg = "No vault found" - raise Exception(msg) +def _vault_exe_filename(platform: str) -> str: + """Filename for the vault binary. + + Args: + platform: The platform `windows`, `linux` or `darwin` + + Returns: + The name of the vault binary (either `vault.exe` or `vault`) + """ + return "vault.exe" if platform == "windows" else "vault" + + +def _vault_url(version: str, platform: str, arch: str = "amd64") -> str: + base = "https://releases.hashicorp.com/vault" + return f"{base}/{version}/vault_{version}_{platform}_{arch}.zip" + + +def _vault_platform() -> str: + return platform.system().lower() -# Package/module global used so that we only install vault once per -# session. -global_vault_dev_exe = VaultDevExe() +def _vault_install_session() -> Path: + tmp = tempfile.TemporaryDirectory(delete=False) + path = vault_download(tmp.name, "1.0.0", _vault_platform()) + os.environ["VAULT_DEV_BIN_PATH"] = str(path) + return path diff --git a/src/vault_dev/server.py b/src/vault_dev/server.py index f6cc896..c089b07 100644 --- a/src/vault_dev/server.py +++ b/src/vault_dev/server.py @@ -35,7 +35,7 @@ def start(self, timeout: float = 5, poll: float = 0.1) -> None: return self._message(f"Starting vault server on port {self.port}") args = [ - self.vault, + str(self.vault), "server", "-dev", "-dev-listen-address", diff --git a/tests/test_install.py b/tests/test_install.py index 7d022fb..987400e 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -4,37 +4,34 @@ import pytest from vault_dev.install import ( - VaultDevExe, + _vault_exe_filename, + _vault_install_session, + _vault_platform, vault_download, - vault_exe_filename, - vault_platform, + vault_path, ) +from vault_dev.utils import transient_envvars def test_vault_exe_name_correct(): - assert vault_exe_filename("windows") == "vault.exe" - assert vault_exe_filename("linux") == "vault" - assert vault_exe_filename("darwin") == "vault" + assert _vault_exe_filename("windows") == "vault.exe" + assert _vault_exe_filename("linux") == "vault" + assert _vault_exe_filename("darwin") == "vault" -def test_vault_download_skips_existing_file(): - platform = vault_platform() +def test_vault_download_skips_existing_file(tmp_path): + platform = _vault_platform() version = "1.0.0" - with tempfile.TemporaryDirectory() as dest: - p = f"{dest}/{vault_exe_filename(platform)}" - open(p, "a").close() - assert vault_download(dest, version, platform) == p - assert os.path.getsize(p) == 0 - - -def test_vault_download(): - v = VaultDevExe() - p = v.install() - assert os.path.exists(p) - assert v.vault() == p - - -def test_error_with_no_suitable_vault(): - v = VaultDevExe() - with pytest.raises(Exception, match="No vault found"): - v.vault() + p = tmp_path / _vault_exe_filename(platform) + open(p, "a").close() + assert vault_download(tmp_path, version, platform) == p + assert os.path.getsize(p) == 0 + + +@pytest.mark.internet +def test_vault_download(tmp_path): + with transient_envvars({"VAULT_DEV_BIN_PATH": None}): + path = _vault_install_session() + assert path.exists() + assert os.environ["VAULT_DEV_BIN_PATH"] == str(path) + assert vault_path() == path