diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 2b07e716f..8845e954a 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -108,13 +108,13 @@ jobs: df -h / # Display available space - name: Run tests - timeout-minutes: 10 + timeout-minutes: 30 env: AMI_NAME: "supabase-postgres-${{ steps.random.outputs.random_string }}" run: | # TODO: use poetry for pkg mgmt pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests - pytest -vv -s testinfra/test_ami_nix.py + pytest -vvvv -s testinfra/test_ami_nix.py - name: Cleanup resources on build cancellation if: ${{ cancelled() }} diff --git a/ansible/files/postgres_prestart.sh.j2 b/ansible/files/postgres_prestart.sh.j2 index 3ffe54c85..40e8debd7 100644 --- a/ansible/files/postgres_prestart.sh.j2 +++ b/ansible/files/postgres_prestart.sh.j2 @@ -26,7 +26,62 @@ update_orioledb_buffers() { fi } +check_extensions_file() { + local extensions_file="/root/pg_extensions.json" + if [ ! -f "$extensions_file" ]; then + echo "extensions: No extensions file found, skipping extensions versions check" + return 1 + fi + return 0 +} + +get_pg_cron_version() { + if ! check_extensions_file; then + return + fi + + local version + version=$(sudo -u postgres /var/lib/postgresql/.nix-profile/bin/jq -r '.pg_cron // empty' "/root/pg_extensions.json") + if [ -z "$version" ]; then + echo "pg_cron: Not specified in extensions file" + return + fi + + if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "pg_cron: Invalid version format: $version" + return + fi + + echo "$version" +} + +switch_pg_cron_version() { + local version="$1" + local switch_script="/var/lib/postgresql/.nix-profile/bin/switch_pg_cron_version" + + if [ ! -x "$switch_script" ]; then + echo "pg_cron: No version switch script available" + return + fi + + echo "pg_cron: Switching to version $version" + sudo -u postgres "$switch_script" "$version" + echo "pg_cron: Version switch completed" +} + +handle_pg_cron_version() { + local version + version=$(get_pg_cron_version) + if [ -n "$version" ]; then + switch_pg_cron_version "$version" + fi +} + main() { + # 1. pg_cron version handling + handle_pg_cron_version + + # 2. orioledb handling local has_orioledb=$(check_orioledb_enabled) if [ "$has_orioledb" -lt 1 ]; then return 0 diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 99b89d6d9..8b67eabe5 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -90,6 +90,12 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" when: stage2_nix + +- name: Install jq from nix binary cache + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#jq" + when: stage2_nix - name: Set ownership and permissions for /etc/ssl/private become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index e57848646..bce0fcd12 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.072-orioledb" - postgres17: "17.4.1.022" - postgres15: "15.8.1.079" + postgresorioledb-17: "17.0.1.067-orioledb-pgcron-4" + postgres17: "17.4.1.022-pgcron-4" + postgres15: "15.8.1.079-pgcron-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/nix/ext/pg_cron-1.3.1-pg15.patch b/nix/ext/pg_cron-1.3.1-pg15.patch new file mode 100644 index 000000000..d3b6cd702 --- /dev/null +++ b/nix/ext/pg_cron-1.3.1-pg15.patch @@ -0,0 +1,31 @@ +diff --git a/src/pg_cron.c b/src/pg_cron.c +index e0ca973..4d51b2c 100644 +--- a/src/pg_cron.c ++++ b/src/pg_cron.c +@@ -14,6 +14,8 @@ + #include + + #include "postgres.h" ++#include "commands/async.h" ++#include "miscadmin.h" + #include "fmgr.h" + + /* these are always necessary for a bgworker */ +@@ -1908,7 +1910,7 @@ CronBackgroundWorker(Datum main_arg) + /* Post-execution cleanup. */ + disable_timeout(STATEMENT_TIMEOUT, false); + CommitTransactionCommand(); +- ProcessCompletedNotifies(); ++ /* ProcessCompletedNotifies removed */ + pgstat_report_activity(STATE_IDLE, command); + pgstat_report_stat(true); + +@@ -2025,7 +2027,7 @@ ExecuteSqlString(const char *sql) + */ + oldcontext = MemoryContextSwitchTo(parsecontext); + #if PG_VERSION_NUM >= 100000 +- querytree_list = pg_analyze_and_rewrite(parsetree, sql, NULL, 0,NULL); ++ querytree_list = pg_analyze_and_rewrite_fixedparams(parsetree, sql, NULL, 0, NULL); + #else + querytree_list = pg_analyze_and_rewrite(parsetree, sql, NULL, 0); + #endif diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix index 792db7676..25121fb8c 100644 --- a/nix/ext/pg_cron.nix +++ b/nix/ext/pg_cron.nix @@ -1,31 +1,147 @@ { lib, stdenv, fetchFromGitHub, postgresql }: -stdenv.mkDerivation rec { - pname = "pg_cron"; - version = "1.6.4"; +let + allVersions = { + "1.3.1" = { + rev = "v1.3.1"; + hash = "sha256-rXotNOtQNmA55ErNxGoNSKZ0pP1uxEVlDGITFHuqGG4="; + patches = [ ./pg_cron-1.3.1-pg15.patch ]; + }; + "1.4.2" = { + rev = "v1.4.2"; + hash = "sha256-P0Fd10Q1p+KrExb35G6otHpc6pD61WnMll45H2jkevM="; + }; + "1.6.4" = { + rev = "v1.6.4"; + hash = "sha256-t1DpFkPiSfdoGG2NgNT7g1lkvSooZoRoUrix6cBID40="; + }; + "1.5.2" = { + rev = "v1.5.2"; + hash = "sha256-+quVWbKJy6wXpL/zwTk5FF7sYwHA7I97WhWmPO/HSZ4="; + }; + }; + + # Simple version string that concatenates all versions with dashes + versionString = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings ["."] ["-"] v) (lib.attrNames allVersions)); + + mkPgCron = pgCronVersion: { rev, hash, patches ? [] }: stdenv.mkDerivation { + pname = "pg_cron"; + version = "${pgCronVersion}-pg${lib.versions.major postgresql.version}"; + + buildInputs = [ postgresql ]; + inherit patches; + + src = fetchFromGitHub { + owner = "citusdata"; + repo = "pg_cron"; + inherit rev hash; + }; - buildInputs = [ postgresql ]; + buildPhase = '' + make PG_CONFIG=${postgresql}/bin/pg_config + + # Create version-specific SQL file + cp pg_cron.sql pg_cron--${pgCronVersion}.sql - src = fetchFromGitHub { - owner = "citusdata"; - repo = pname; - rev = "v${version}"; - hash = "sha256-t1DpFkPiSfdoGG2NgNT7g1lkvSooZoRoUrix6cBID40="; + # Create versioned control file with modified module path + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/pg_cron'|" \ + pg_cron.control > pg_cron--${pgCronVersion}.control + ''; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension,bin} + + # Install versioned library + install -Dm755 pg_cron${postgresql.dlSuffix} $out/lib/pg_cron-${pgCronVersion}${postgresql.dlSuffix} + + # Install version-specific files + install -Dm644 pg_cron--${pgCronVersion}.sql $out/share/postgresql/extension/ + install -Dm644 pg_cron--${pgCronVersion}.control $out/share/postgresql/extension/ + + # Install upgrade scripts + find . -name 'pg_cron--*--*.sql' -exec install -Dm644 {} $out/share/postgresql/extension/ \; + ''; }; + getVersions = pg: + if lib.versionAtLeast pg.version "17" + then { "1.6.4" = allVersions."1.6.4"; } + else allVersions; + + allVersionsForPg = lib.mapAttrs mkPgCron (getVersions postgresql); + +in +stdenv.mkDerivation { + pname = "pg_cron-all"; + version = versionString; + + buildInputs = lib.attrValues allVersionsForPg; + + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + installPhase = '' - mkdir -p $out/{lib,share/postgresql/extension} + mkdir -p $out/{lib,share/postgresql/extension,bin} + + # Install all versions + for drv in ${lib.concatStringsSep " " (lib.attrValues allVersionsForPg)}; do + ln -sv $drv/lib/* $out/lib/ + cp -v --no-clobber $drv/share/postgresql/extension/* $out/share/postgresql/extension/ || true + done + + # Create default symlinks + latest_control=$(ls -v $out/share/postgresql/extension/pg_cron--*.control | tail -n1) + latest_version=$(basename "$latest_control" | sed -E 's/pg_cron--([0-9.]+).control/\1/') + + # Create main control file with default_version + echo "default_version = '$latest_version'" > $out/share/postgresql/extension/pg_cron.control + cat "$latest_control" >> $out/share/postgresql/extension/pg_cron.control + + # Library symlink + ln -sfnv pg_cron-$latest_version${postgresql.dlSuffix} $out/lib/pg_cron${postgresql.dlSuffix} + + # Create version switcher script + cat > $out/bin/switch_pg_cron_version <<'EOF' + #!/bin/sh + set -e + + if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "Example: $0 1.4.2" + exit 1 + fi + + VERSION=$1 + LIB_DIR=$(dirname "$0")/../lib + + # Use platform-specific extension + if [ "$(uname)" = "Darwin" ]; then + EXT=".dylib" + else + EXT=".so" + fi + + # Check if version exists + if [ ! -f "$LIB_DIR/pg_cron-$VERSION$EXT" ]; then + echo "Error: Version $VERSION not found" + exit 1 + fi + + # Update library symlink + ln -sfnv "pg_cron-$VERSION$EXT" "$LIB_DIR/pg_cron$EXT" + + echo "Successfully switched pg_cron to version $VERSION" + EOF - cp *${postgresql.dlSuffix} $out/lib - cp *.sql $out/share/postgresql/extension - cp *.control $out/share/postgresql/extension + chmod +x $out/bin/switch_pg_cron_version ''; meta = with lib; { - description = "Run Cron jobs through PostgreSQL"; - homepage = "https://github.com/citusdata/pg_cron"; - changelog = "https://github.com/citusdata/pg_cron/raw/v${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + description = "Run Cron jobs through PostgreSQL (multi-version compatible)"; + homepage = "https://github.com/citusdata/pg_cron"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; }; } diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 4d354fac3..8d236b7a3 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -10,6 +10,7 @@ from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey from time import sleep +from typing import Tuple, Dict # if GITHUB_RUN_ID is not set, use a default value that includes the user and hostname RUN_ID = os.environ.get( @@ -229,6 +230,7 @@ def gzip_then_base64_encode(s: str) -> str: runcmd: - 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt' - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .' + - 'if [ "$POSTGRES_MAJOR_VERSION" = "15" ]; then echo \'{{"pg_cron":"1.3.1"}}\' | sudo tee /root/pg_extensions.json && sudo chmod 644 /root/pg_extensions.json; fi' - 'bash init.sh "staging"' - 'rm -rf /tmp/*' """, @@ -291,11 +293,46 @@ def get_ssh_connection(instance_ip, ssh_identity_file, max_retries=10): temp_key.get_priv_key_file(), ) - def is_healthy(host, instance_ip, ssh_identity_file) -> bool: + def run_detailed_checks(host): + logger.info("Running detailed system checks...") + + # Log Nix profile setup checks + logger.info("Checking Nix profile setup:") + nix_profile_result = host.run("ls -la /var/lib/postgresql/.nix-profile") + logger.info(f"Nix profile directory:\n{nix_profile_result.stdout}\n{nix_profile_result.stderr}") + + nix_bin_result = host.run("ls -la /var/lib/postgresql/.nix-profile/bin") + logger.info(f"Nix profile bin directory:\n{nix_bin_result.stdout}\n{nix_bin_result.stderr}") + + # Check PostgreSQL logs directory + logger.info("Checking PostgreSQL logs directory:") + result = host.run("sudo ls -la /var/log/postgresql/") + logger.info(f"log directory contents:\n{result.stdout}\n{result.stderr}") + + # Check any existing PostgreSQL logs + logger.info("Checking existing PostgreSQL logs:") + result = host.run("sudo cat /var/log/postgresql/*.log") + logger.info(f"postgresql logs:\n{result.stdout}\n{result.stderr}") + + # Check the startup log + logger.info("PostgreSQL startup log:") + result = host.run(f"sudo cat {startup_log}") + logger.info(f"startup log contents:\n{result.stdout}\n{result.stderr}") + + # Check PostgreSQL environment + logger.info("PostgreSQL environment:") + result = host.run("sudo -u postgres env | grep POSTGRES") + logger.info(f"postgres environment:\n{result.stdout}\n{result.stderr}") + + def is_healthy(host, instance_ip, ssh_identity_file) -> Tuple[bool, Dict[str, bool]]: + service_status = {} # Track status of each service + health_checks = [ ( "postgres", - lambda h: h.run("sudo -u postgres /usr/bin/pg_isready -U postgres"), + lambda h: ( + h.run("sudo -u postgres /usr/bin/pg_isready -U postgres") + ), ), ( "adminapi", @@ -321,26 +358,71 @@ def is_healthy(host, instance_ip, ssh_identity_file) -> bool: for service, check in health_checks: try: - cmd = check(host) - if cmd.failed is True: - logger.warning(f"{service} not ready") - return False - except Exception: - logger.warning( - f"Connection failed during {service} check, attempting reconnect..." - ) + if service == "postgres": + pg_isready = check(host) + + # Always read and log the PostgreSQL logs + logger.warning("PostgreSQL status check:") + try: + log_files = [ + "/var/log/postgresql/*.log", + "/var/log/postgresql/*.csv" + ] + + for log_pattern in log_files: + log_result = host.run(f"sudo cat {log_pattern}") + if not log_result.failed: + logger.error(f"PostgreSQL logs from {log_pattern}:") + logger.error(log_result.stdout) + if log_result.stderr: + logger.error(f"Log read errors: {log_result.stderr}") + else: + logger.error(f"Failed to read PostgreSQL logs from {log_pattern}: {log_result.stderr}") + except Exception as e: + logger.error(f"Error reading PostgreSQL logs: {str(e)}") + + service_status[service] = not pg_isready.failed + + else: + cmd = check(host) + service_status[service] = not cmd.failed + if cmd.failed: + logger.warning(f"{service} not ready") + logger.error(f"{service} command failed with rc={cmd.rc}") + logger.error(f"{service} stdout: {cmd.stdout}") + logger.error(f"{service} stderr: {cmd.stderr}") + + except Exception as e: + logger.warning(f"Connection failed during {service} check, attempting reconnect...") + logger.error(f"Error details: {str(e)}") host = get_ssh_connection(instance_ip, ssh_identity_file) - return False + service_status[service] = False + + # Log overall status of all services + logger.info("Service health status:") + for service, healthy in service_status.items(): + logger.info(f"{service}: {'healthy' if healthy else 'unhealthy'}") - return True + # If any service is unhealthy, wait and return False with status + if not all(service_status.values()): + if service_status.get("postgres", False): # If postgres is healthy but others aren't + sleep(5) # Only wait if postgres is up but other services aren't + logger.warning("Some services are not healthy, will retry...") + return False, service_status + + logger.info("All services are healthy, proceeding to tests...") + return True, service_status while True: - if is_healthy( + healthy, status = is_healthy( host=host, instance_ip=instance.public_ip_address, ssh_identity_file=temp_key.get_priv_key_file(), - ): + ) + if healthy: + logger.info("Health check passed, starting tests...") break + logger.warning(f"Health check failed, service status: {status}") sleep(1) # return a testinfra connection to the instance @@ -476,3 +558,32 @@ def test_postgrest_ending_empty_key_query_parameter_is_removed(host): }, ) assert res.ok + + +def test_pg_cron_extension(host): + # Only run this test for PostgreSQL 15 + postgres_version = os.environ.get("POSTGRES_MAJOR_VERSION") + if postgres_version != "15": + pytest.skip(f"Skipping pg_cron test for PostgreSQL version {postgres_version}") + + # Connect as supabase_admin and create the extension + with host.sudo("postgres"): + result = host.run('psql -U supabase_admin -d postgres -c "CREATE EXTENSION pg_cron WITH SCHEMA pg_catalog VERSION \'1.3.1\';"') + assert result.rc == 0, f"Failed to create pg_cron extension: {result.stderr}" + + # Create test table + result = host.run('psql -U supabase_admin -d postgres -c "CREATE TABLE cron_test_log (id SERIAL PRIMARY KEY, message TEXT, log_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP);"') + assert result.rc == 0, f"Failed to create test table: {result.stderr}" + + # Schedule a job + result = host.run('psql -U supabase_admin -d postgres -c "SELECT cron.schedule(\'* * * * *\', \'INSERT INTO cron_test_log (message) VALUES (\\\'Hello from pg_cron!\\\');\');"') + assert result.rc == 0, f"Failed to schedule job: {result.stderr}" + assert "1" in result.stdout, "Expected schedule ID 1" + + # Verify job is scheduled + result = host.run('psql -U supabase_admin -d postgres -c "SELECT * FROM cron.job;"') + assert result.rc == 0, f"Failed to query cron.job: {result.stderr}" + assert "* * * * *" in result.stdout, "Expected cron schedule pattern" + assert "INSERT INTO cron_test_log" in result.stdout, "Expected cron command" + assert "postgres" in result.stdout, "Expected postgres username" + assert "postgres" in result.stdout, "Expected postgres database"