From 9ed045528e1d88b1cbf9f307f26f930662041c4b Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sat, 11 Jan 2025 23:49:51 +0100 Subject: [PATCH 1/8] add ansible/lint --- Makefile | 6 ++++++ deploy/playbooks/01_setup.yml | 34 +++++++++++++++++----------------- deploy/playbooks/02_nginx.yml | 16 +++++++++++----- deploy/playbooks/03_app.yml | 21 ++++++++++++--------- 4 files changed, 46 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index dabbabb..2a7d059 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ CI_RUN=cd intbot && DJANGO_SETTINGS_MODULE="intbot.settings" DJANGO_ENV="ci" # Deployment DEPLOY_CMD=cd deploy && uvx --from "ansible-core" ansible-playbook -i hosts.yml +DEPLOY_LINT_CMD=cd deploy && uvx --from "ansible-lint" ansible-lint # mostly useful for docker and deployment current_git_hash=$(shell git rev-parse HEAD) @@ -149,3 +150,8 @@ deploy/provision: deploy/app: @echo "Deploying version $(V) to a remote server" $(DEPLOY_CMD) playbooks/03_app.yml --extra-vars "app_version=$(V)" + +deploy/lint: + $(DEPLOY_LINT_CMD) playbooks/01_setup.yml + $(DEPLOY_LINT_CMD) playbooks/02_nginx.yml + $(DEPLOY_LINT_CMD) playbooks/03_app.yml diff --git a/deploy/playbooks/01_setup.yml b/deploy/playbooks/01_setup.yml index 4768bf3..bcb0743 100644 --- a/deploy/playbooks/01_setup.yml +++ b/deploy/playbooks/01_setup.yml @@ -1,14 +1,14 @@ - name: Deploy nginx and Let's Encrypt SSL certificate hosts: intbot_setup - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - name: Install Docker dependencies - apt: + ansible.builtin.apt: name: "{{ package }}" state: present - update_cache: yes + update_cache: true vars: package: - apt-transport-https @@ -21,22 +21,22 @@ - name: Install Docker block: - name: Add Docker GPG key - apt_key: + ansible.builtin.apt_key: url: https://download.docker.com/linux/ubuntu/gpg state: present - name: Add Docker repository - apt_repository: + ansible.builtin.apt_repository: repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable state: present - name: Install Docker - apt: + ansible.builtin.apt: name: docker-ce state: present - name: Combine non-root users to a single list - set_fact: + ansible.builtin.set_fact: non_root_user_names: ["{{ nginx_user }}", "{{ app_user }}"] - name: Create non-root users @@ -45,10 +45,10 @@ ansible.builtin.user: name: "{{ username }}" shell: "/bin/bash" - generate_ssh_key: yes + generate_ssh_key: true ssh_key_type: ed25519 ssh_key_comment: "{{ username }}@{{ inventory_hostname }}" - create_home: yes + create_home: true loop: "{{ non_root_user_names }}" loop_control: loop_var: username @@ -59,37 +59,37 @@ state: directory owner: "{{ username }}" group: "{{ username }}" + mode: "0755" loop: "{{ non_root_user_names }}" loop_control: loop_var: username - name: Then copy the authorized_keys from root so you can ssh later to the user - copy: + ansible.builtin.copy: src: "/root/.ssh/authorized_keys" dest: "/home/{{ username }}/.ssh/authorized_keys" owner: "{{ username }}" group: "{{ username }}" mode: "0600" - remote_src: "yes" + remote_src: true loop: "{{ non_root_user_names }}" loop_control: loop_var: username - name: Add the non root users (both nginx and app) to docker group - user: + ansible.builtin.user: name: "{{ username }}" groups: docker - append: yes + append: true loop: "{{ non_root_user_names }}" loop_control: loop_var: username - name: Read the deploy public key - slurp: + ansible.builtin.slurp: src: "/home/{{ app_user }}/.ssh/id_ed25519.pub" register: deploy_key - name: Display the public key - debug: + ansible.builtin.debug: msg: "For private repositories, make sure to put this key as deploy key on github: {{ deploy_key.content | b64decode }}" - diff --git a/deploy/playbooks/02_nginx.yml b/deploy/playbooks/02_nginx.yml index d2b4a60..2422783 100644 --- a/deploy/playbooks/02_nginx.yml +++ b/deploy/playbooks/02_nginx.yml @@ -4,22 +4,28 @@ tasks: - name: Copy nginx configuration file ansible.builtin.template: - src: ../templates/nginx/nginx.conf.j2 + src: nginx/nginx.conf.j2 dest: ./nginx.conf + mode: "0644" - name: Create a server Makefile (for nginx) to manage on-server tasks ansible.builtin.template: - src: ../templates/nginx/Makefile.nginx.j2 + src: nginx/Makefile.nginx.j2 dest: ./Makefile + mode: "0644" - name: Set up docker-compose.yml on the remote server ansible.builtin.template: - src: ../templates/nginx/docker-compose.nginx.yml.j2 + src: nginx/docker-compose.nginx.yml.j2 dest: ./docker-compose.yml + mode: "0644" - name: Make sure the directory structure for certs exist - shell: mkdir -p ./data/certbot/conf + ansible.builtin.file: + path: "/home/{{ ansible_user }}/data/cerbot/conf" + state: directory + mode: "0755" - name: Display info at the end - debug: + ansible.builtin.debug: msg: "Go to /home/{{ ansible_user }} and run make certbot/init-staging; then make certbot/upgrade-to-prod" diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index eea96b3..663699e 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -3,26 +3,28 @@ tasks: - name: Clone the repository to a specific version (to a temp location) - git: + ansible.builtin.git: repo: "{{ repository_url }}" dest: /tmp/src - accept_hostkey: yes + accept_hostkey: true version: "{{ app_version }}" - name: Build with a given commit hash # This will be stored in local registry, and available as version to docker-compose # where we can just reference correct version - shell: "cd /tmp/src && make docker/build V={{ app_version }}" + ansible.builtin.shell: "cd /tmp/src && make docker/build V={{ app_version }}" - name: Create a server Makefile to manage app tasks ansible.builtin.template: - src: ../templates/app/Makefile.app.j2 + src: app/Makefile.app.j2 dest: ./Makefile + mode: "0644" - name: Set up docker-compose.yml for the app ansible.builtin.template: - src: ../templates/app/docker-compose.app.yml.j2 + src: app/docker-compose.app.yml.j2 dest: ./docker-compose.yml + mode: "0644" - name: Check if the env file exists ansible.builtin.stat: @@ -31,8 +33,9 @@ - name: If env file doesn't exist - copy the example ansible.builtin.copy: - src: ../templates/app/intbot.env.example + src: app/intbot.env.example dest: intbot.env.example + mode: "0644" when: not env_file.stat.exists - name: If the env file doesn't exist - fail with error message @@ -41,10 +44,10 @@ when: not env_file.stat.exists - name: Start docker compose to see if everything is running - shell: "docker compose up -d" + ansible.builtin.shell: "docker compose up -d" - name: Migrate on prod - shell: "make prod/migrate" + ansible.builtin.shell: "make prod/migrate" - name: Restart everything and finish - shell: "docker compose up -d" + ansible.builtin.shell: "docker compose up -d" From ec78bcf95f37997c235bf6d4fe24a3492fceebf0 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sat, 18 Jan 2025 00:40:20 +0100 Subject: [PATCH 2/8] Update deploy/playbooks/02_nginx.yml Co-authored-by: Cyril Bitterich --- deploy/playbooks/02_nginx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/playbooks/02_nginx.yml b/deploy/playbooks/02_nginx.yml index 2422783..223d76a 100644 --- a/deploy/playbooks/02_nginx.yml +++ b/deploy/playbooks/02_nginx.yml @@ -22,7 +22,7 @@ - name: Make sure the directory structure for certs exist ansible.builtin.file: - path: "/home/{{ ansible_user }}/data/cerbot/conf" + path: "/home/{{ ansible_user }}/data/certbot/conf" state: directory mode: "0755" From b06bc8bba0271d1ecb2cbbe100ff9c489dc2e775 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sat, 18 Jan 2025 00:41:11 +0100 Subject: [PATCH 3/8] Update deploy/playbooks/03_app.yml Co-authored-by: Cyril Bitterich --- deploy/playbooks/03_app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 663699e..f2991cc 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -18,7 +18,7 @@ ansible.builtin.template: src: app/Makefile.app.j2 dest: ./Makefile - mode: "0644" + mode: "0640" - name: Set up docker-compose.yml for the app ansible.builtin.template: From 1a06bb59b559596078ff42e590f18694dafe4c70 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 17:03:34 +0100 Subject: [PATCH 4/8] Apply suggestions from code review Co-authored-by: Cyril Bitterich --- deploy/playbooks/02_nginx.yml | 4 ++-- deploy/playbooks/03_app.yml | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/deploy/playbooks/02_nginx.yml b/deploy/playbooks/02_nginx.yml index 223d76a..884dbad 100644 --- a/deploy/playbooks/02_nginx.yml +++ b/deploy/playbooks/02_nginx.yml @@ -12,13 +12,13 @@ ansible.builtin.template: src: nginx/Makefile.nginx.j2 dest: ./Makefile - mode: "0644" + mode: "0640" - name: Set up docker-compose.yml on the remote server ansible.builtin.template: src: nginx/docker-compose.nginx.yml.j2 dest: ./docker-compose.yml - mode: "0644" + mode: "0640" - name: Make sure the directory structure for certs exist ansible.builtin.file: diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index f2991cc..167bc24 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -12,7 +12,9 @@ - name: Build with a given commit hash # This will be stored in local registry, and available as version to docker-compose # where we can just reference correct version - ansible.builtin.shell: "cd /tmp/src && make docker/build V={{ app_version }}" + ansible.builtin.shell: + chdir: /tmp/src + cmd: "/usr/bin/make docker/build V={{ app_version }}" - name: Create a server Makefile to manage app tasks ansible.builtin.template: @@ -24,7 +26,7 @@ ansible.builtin.template: src: app/docker-compose.app.yml.j2 dest: ./docker-compose.yml - mode: "0644" + mode: "0640" - name: Check if the env file exists ansible.builtin.stat: @@ -44,10 +46,16 @@ when: not env_file.stat.exists - name: Start docker compose to see if everything is running - ansible.builtin.shell: "docker compose up -d" + ansible.builtin.command: + chdir: {{ ansible_user_dir }} + cmd: "docker compose up -d" - name: Migrate on prod - ansible.builtin.shell: "make prod/migrate" + ansible.builtin.shell: + chdir: /tmp/src + cmd: "/usr/bin/make prod/migrate" - name: Restart everything and finish - ansible.builtin.shell: "docker compose up -d" + ansible.builtin.command: + chdir: {{ ansible_user_dir }} + cmd: "docker compose up -d" From 4d87fb420eaecdc11065733d70f0b336c64644c0 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 17:12:56 +0100 Subject: [PATCH 5/8] tweak ansible lint --- Makefile | 2 +- deploy/playbooks/03_app.yml | 10 +++++----- deploy/{ => playbooks}/templates/app/.env.example | 0 deploy/{ => playbooks}/templates/app/Makefile.app.j2 | 0 .../templates/app/docker-compose.app.yml.j2 | 0 .../{ => playbooks}/templates/app/intbot.env.example | 0 .../{ => playbooks}/templates/nginx/Makefile.nginx.j2 | 0 .../templates/nginx/docker-compose.nginx.yml.j2 | 0 deploy/{ => playbooks}/templates/nginx/nginx.conf.j2 | 0 9 files changed, 6 insertions(+), 6 deletions(-) rename deploy/{ => playbooks}/templates/app/.env.example (100%) rename deploy/{ => playbooks}/templates/app/Makefile.app.j2 (100%) rename deploy/{ => playbooks}/templates/app/docker-compose.app.yml.j2 (100%) rename deploy/{ => playbooks}/templates/app/intbot.env.example (100%) rename deploy/{ => playbooks}/templates/nginx/Makefile.nginx.j2 (100%) rename deploy/{ => playbooks}/templates/nginx/docker-compose.nginx.yml.j2 (100%) rename deploy/{ => playbooks}/templates/nginx/nginx.conf.j2 (100%) diff --git a/Makefile b/Makefile index 2a7d059..30db2d4 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ deploy/app: @echo "Deploying version $(V) to a remote server" $(DEPLOY_CMD) playbooks/03_app.yml --extra-vars "app_version=$(V)" -deploy/lint: +lint/deploy: $(DEPLOY_LINT_CMD) playbooks/01_setup.yml $(DEPLOY_LINT_CMD) playbooks/02_nginx.yml $(DEPLOY_LINT_CMD) playbooks/03_app.yml diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 167bc24..084c64b 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -12,7 +12,7 @@ - name: Build with a given commit hash # This will be stored in local registry, and available as version to docker-compose # where we can just reference correct version - ansible.builtin.shell: + ansible.builtin.command: chdir: /tmp/src cmd: "/usr/bin/make docker/build V={{ app_version }}" @@ -47,15 +47,15 @@ - name: Start docker compose to see if everything is running ansible.builtin.command: - chdir: {{ ansible_user_dir }} + chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" - name: Migrate on prod - ansible.builtin.shell: - chdir: /tmp/src + ansible.builtin.command: + chdir: "{{ ansible_user_dir }}" cmd: "/usr/bin/make prod/migrate" - name: Restart everything and finish ansible.builtin.command: - chdir: {{ ansible_user_dir }} + chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" diff --git a/deploy/templates/app/.env.example b/deploy/playbooks/templates/app/.env.example similarity index 100% rename from deploy/templates/app/.env.example rename to deploy/playbooks/templates/app/.env.example diff --git a/deploy/templates/app/Makefile.app.j2 b/deploy/playbooks/templates/app/Makefile.app.j2 similarity index 100% rename from deploy/templates/app/Makefile.app.j2 rename to deploy/playbooks/templates/app/Makefile.app.j2 diff --git a/deploy/templates/app/docker-compose.app.yml.j2 b/deploy/playbooks/templates/app/docker-compose.app.yml.j2 similarity index 100% rename from deploy/templates/app/docker-compose.app.yml.j2 rename to deploy/playbooks/templates/app/docker-compose.app.yml.j2 diff --git a/deploy/templates/app/intbot.env.example b/deploy/playbooks/templates/app/intbot.env.example similarity index 100% rename from deploy/templates/app/intbot.env.example rename to deploy/playbooks/templates/app/intbot.env.example diff --git a/deploy/templates/nginx/Makefile.nginx.j2 b/deploy/playbooks/templates/nginx/Makefile.nginx.j2 similarity index 100% rename from deploy/templates/nginx/Makefile.nginx.j2 rename to deploy/playbooks/templates/nginx/Makefile.nginx.j2 diff --git a/deploy/templates/nginx/docker-compose.nginx.yml.j2 b/deploy/playbooks/templates/nginx/docker-compose.nginx.yml.j2 similarity index 100% rename from deploy/templates/nginx/docker-compose.nginx.yml.j2 rename to deploy/playbooks/templates/nginx/docker-compose.nginx.yml.j2 diff --git a/deploy/templates/nginx/nginx.conf.j2 b/deploy/playbooks/templates/nginx/nginx.conf.j2 similarity index 100% rename from deploy/templates/nginx/nginx.conf.j2 rename to deploy/playbooks/templates/nginx/nginx.conf.j2 From 6544f7b40c0bd6096e764ad1650f6dee0b0b1d22 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 20:01:23 +0100 Subject: [PATCH 6/8] lint deployment playbooks on CI --- .github/workflows/build_and_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index d6280de..e7c1212 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -42,6 +42,7 @@ jobs: run: | docker run --rm intbot make ci/lint docker run --rm intbot make ci/type-check + docker run --rm intbot make lint/deploy services: postgres: From d7496b5161ecf6dc0321c4c4c10f513c66b04cc3 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 20:04:52 +0100 Subject: [PATCH 7/8] deploy is not pat of the image, so lets run it with uvx --- .github/workflows/build_and_deploy.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index e7c1212..9b32f9a 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -14,6 +14,10 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + # Uv is needed for the deployment lint + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -42,7 +46,10 @@ jobs: run: | docker run --rm intbot make ci/lint docker run --rm intbot make ci/type-check - docker run --rm intbot make lint/deploy + + + - name: Run deployment playbooks lint + run: make lint/deploy services: postgres: From e411347c6dbd62aa025ea2eb642dadd7e9270c93 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 20:11:04 +0100 Subject: [PATCH 8/8] fix remaining ansible lint errors --- deploy/playbooks/03_app.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 084c64b..85e1c0a 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -15,6 +15,9 @@ ansible.builtin.command: chdir: /tmp/src cmd: "/usr/bin/make docker/build V={{ app_version }}" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 - name: Create a server Makefile to manage app tasks ansible.builtin.template: @@ -49,13 +52,22 @@ ansible.builtin.command: chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 - name: Migrate on prod ansible.builtin.command: chdir: "{{ ansible_user_dir }}" cmd: "/usr/bin/make prod/migrate" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 - name: Restart everything and finish ansible.builtin.command: chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0