commit 9a5d1bddec16428dcb47fcb70aac6506ae6faa48 Author: ace Date: Wed May 15 15:52:33 2024 +0300 first commit diff --git a/defaults/main.yaml b/defaults/main.yaml new file mode 100644 index 0000000..b08db73 --- /dev/null +++ b/defaults/main.yaml @@ -0,0 +1,53 @@ +wal_g_version: "3.0.0" +wal_g_install_repo: true + +wal_g_pg: false +wal_g_pg_binary_name: "wal-g-pg" +wal_g_pg_user: "postgres" +wal_g_pg_database: "postgres" +wal_g_restore: false +wal_g_config_name: ".walg.json" +wal_g_restore_config_name: ".walg-restore.json" +wal_g_pg_config_owner: "postgres" +wal_g_pg_config_group: "postgres" +wal_g_pg_backup_owner: "postgres" +wal_g_pg_backup_group: "postgres" +wal_g_pg_major_version: "15" + +wal_g_backup_enable: true +wal_g_backup_schedule: "" +wal_g_backup_retention_schedule: "" +wal_g_backup_retention_number: 7 +wal_g_backup_permanent_schedule: "" + +# If backuping through proxy +#wal_g_http_proxy: '' +#wal_g_https_proxy: '' + +wal_g_postgresql_supported_versions: + - 13 + - 14 + - 15 + - 16 + +wal_g_default_config: # more options see https://github.com/wal-g/wal-g#configuration +# AWS_ACCESS_KEY_ID: "secret-key" +# AWS_SECRET_ACCESS_KEY: "access-key" +# AWS_ENDPOINT: "https://s3.example.com" +# WALG_S3_PREFIX: "s3://bucket" +# AWS_S3_FORCE_PATH_STYLE: "true" + WALG_COMPRESSION_METHOD: "brotli" + PGDATA: "{{ wal_g_pg_data_dir }}" + PGHOST: "/var/run/postgresql" + PGUSER: "{{ wal_g_pg_user }}" + PGDATABASE: "{{ wal_g_pg_database }}" +# WALG_LIBSODIUM_KEY: "d82560ddab2b096c56ea8c6573dfdf29de8cd6f2bb5e672d1d3b5a89749b8cbd" + WALG_LIBSODIUM_KEY_TRANSFORM: "hex" + +wal_g_default_restore_config: + WALG_COMPRESSION_METHOD: "brotli" + PGDATA: "{{ wal_g_pg_data_dir }}" + PGHOST: "/var/run/postgresql" + PGUSER: "{{ wal_g_pg_user }}" + PGDATABASE: "{{ wal_g_pg_database }}" + WALG_LIBSODIUM_KEY_TRANSFORM: "hex" diff --git a/handlers/main.yaml b/handlers/main.yaml new file mode 100644 index 0000000..2ddb054 --- /dev/null +++ b/handlers/main.yaml @@ -0,0 +1,50 @@ +- name: Daemon reload backup service + systemd: + name: backup.service + enabled: no + masked : no + daemon_reload: yes + when: backup_service.changed + +- name: Restart backup timer + systemd: + name: backup.timer + enabled: yes + masked : no + state: started + daemon_reload: yes + when: backup_timer.changed + +- name: Daemon reload backup retention service + systemd: + name: backup-retention.service + enabled: no + masked : no + daemon_reload: yes + when: backup_retention_service.changed + +- name: Restart backup retention timer + systemd: + name: backup-retention.timer + enabled: yes + masked : no + state: started + daemon_reload: yes + when: backup_retention_timer.changed + +- name: Daemon reload backup permanent service + systemd: + name: backup-permanent.service + enabled: no + masked : no + daemon_reload: yes + when: backup_permanent_service.changed + +- name: Restart backup permanent timer + systemd: + name: backup-permanent.timer + enabled: yes + masked : no + state: started + daemon_reload: yes + when: backup_permanent_timer.changed diff --git a/meta/main.yaml b/meta/main.yaml new file mode 100644 index 0000000..083d8c1 --- /dev/null +++ b/meta/main.yaml @@ -0,0 +1,7 @@ +--- +galaxy_info: + description: WAL-G backup util + galaxy_tags: + - wal-g + +dependencies: [] diff --git a/tasks/backup_job.yaml b/tasks/backup_job.yaml new file mode 100644 index 0000000..6cff2a5 --- /dev/null +++ b/tasks/backup_job.yaml @@ -0,0 +1,22 @@ +- name: Template backup script + template: + src: backup.sh.j2 + dest: "{{ wal_g_pg_home_dir }}/backup.sh" + mode: 0755 + owner: "{{ wal_g_pg_backup_owner }}" + group: "{{ wal_g_pg_backup_group }}" + +- name: Template systemd backup service + template: + src: backup.service.j2 + dest: /etc/systemd/system/backup.service + notify: Daemon reload backup service + register: backup_service + +- name: Reload systemd backup service + systemd: + name: backup.service + enabled: no + masked: no + daemon_reload: yes + when: not backup_service.changed diff --git a/tasks/backup_job_timer.yaml b/tasks/backup_job_timer.yaml new file mode 100644 index 0000000..5e6bf4a --- /dev/null +++ b/tasks/backup_job_timer.yaml @@ -0,0 +1,15 @@ +- name: Template systemd backup timer + template: + src: backup.timer.j2 + dest: /etc/systemd/system/backup.timer + notify: Restart backup timer + register: backup_timer + +- name: Enable and start systemd backup timer + systemd: + name: backup.timer + enabled: "{{ 'yes' if wal_g_backup_enable else 'no' }}" + masked: no + state: started + daemon_reload: yes + when: not backup_timer.changed diff --git a/tasks/backup_permanent_job.yaml b/tasks/backup_permanent_job.yaml new file mode 100644 index 0000000..d1e23cc --- /dev/null +++ b/tasks/backup_permanent_job.yaml @@ -0,0 +1,22 @@ +- name: Template backup permanent script + template: + src: backup-permanent.sh.j2 + dest: "{{ wal_g_pg_home_dir }}/backup-permanent.sh" + mode: 0755 + owner: "{{ wal_g_pg_backup_owner }}" + group: "{{ wal_g_pg_backup_group }}" + +- name: Template systemd backup permanent service + template: + src: backup-permanent.service.j2 + dest: /etc/systemd/system/backup-permanent.service + notify: Daemon reload backup permanent service + register: backup_permanent_service + +- name: Reload systemd backup permanent service + systemd: + name: backup-permanent.service + enabled: no + masked: no + daemon_reload: yes + when: not backup_permanent_service.changed diff --git a/tasks/backup_permanent_job_timer.yaml b/tasks/backup_permanent_job_timer.yaml new file mode 100644 index 0000000..bbca385 --- /dev/null +++ b/tasks/backup_permanent_job_timer.yaml @@ -0,0 +1,15 @@ +- name: Template systemd backup permanent timer + template: + src: backup-permanent.timer.j2 + dest: /etc/systemd/system/backup-permanent.timer + notify: Restart backup permanent timer + register: backup_permanent_timer + +- name: Enable and start systemd backup permanent timer + systemd: + name: backup-permanent.timer + enabled: "{{ 'yes' if wal_g_backup_enable else 'no' }}" + masked: no + state: started + daemon_reload: yes + when: not backup_permanent_timer.changed diff --git a/tasks/backup_retention_job.yaml b/tasks/backup_retention_job.yaml new file mode 100644 index 0000000..6ec98b5 --- /dev/null +++ b/tasks/backup_retention_job.yaml @@ -0,0 +1,23 @@ +- name: Template backup retention script + template: + src: backup-retention.sh.j2 + dest: "{{ wal_g_pg_home_dir }}/backup-retention.sh" + mode: 0755 + owner: "{{ wal_g_pg_backup_owner }}" + group: "{{ wal_g_pg_backup_group }}" + +- name: Template systemd backup retention service + template: + src: backup-retention.service.j2 + dest: /etc/systemd/system/backup-retention.service + notify: Daemon reload backup retention service + register: backup_retention_service + +- name: Reload systemd backup retention service + systemd: + name: backup-retention.service + enabled: no + masked: no + daemon_reload: yes + when: not backup_retention_service.changed + diff --git a/tasks/backup_retention_job_timer.yaml b/tasks/backup_retention_job_timer.yaml new file mode 100644 index 0000000..dae7580 --- /dev/null +++ b/tasks/backup_retention_job_timer.yaml @@ -0,0 +1,15 @@ +- name: Template systemd backup retention timer + template: + src: backup-retention.timer.j2 + dest: /etc/systemd/system/backup-retention.timer + notify: Restart backup retention timer + register: backup_retention_timer + +- name: Enable and start systemd backup retention timer + systemd: + name: backup-retention.timer + enabled: "{{ 'yes' if wal_g_backup_enable else 'no' }}" + masked: no + state: started + daemon_reload: yes + when: not backup_retention_timer.changed diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..d600c96 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,56 @@ +--- +- name: Load a variable file based on the OS type + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: + - "{{ ansible_facts['distribution'] }}.yaml" + - "{{ ansible_facts['os_family'] }}.yaml" + paths: + - "vars" + tags: wal_g_vars + +- name: Set facts about needed vars + set_fact: + wal_g_pg_home_dir: "{{ wal_g_pg_home_dir }}" + wal_g_pg_data_dir: "{{ wal_g_pg_data_dir }}" + +- name: "Include install WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: pg.yaml + when: wal_g_pg + +- name: "Include backup job with WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: backup_job.yaml + when: + - wal_g_pg + +- name: "Include backup job timer with WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: backup_job_timer.yaml + when: + - wal_g_pg + - wal_g_backup_schedule is defined + - wal_g_backup_schedule | length > 0 + +- name: "Include backup retention job with WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: backup_retention_job.yaml + when: + - wal_g_pg + +- name: "Include backup retention job timer with WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: backup_retention_job_timer.yaml + when: + - wal_g_pg + - wal_g_backup_retention_schedule is defined + - wal_g_backup_retention_schedule | length > 0 + +- name: "Include permanent backup job with WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: backup_permanent_job.yaml + when: + - wal_g_pg + +- name: "Include permanent backup job timer with WAL-G {{ wal_g_version }} for PostgreSQL" + include_tasks: backup_permanent_job_timer.yaml + when: + - wal_g_pg + - wal_g_backup_permanent_schedule is defined + - wal_g_backup_permanent_schedule | length > 0 diff --git a/tasks/pg.yaml b/tasks/pg.yaml new file mode 100644 index 0000000..3723e61 --- /dev/null +++ b/tasks/pg.yaml @@ -0,0 +1,111 @@ +--- +- name: RHEL based distros + when: ansible_os_family == "RedHat" + block: + - name: Install WAL-G repository + ansible.builtin.yum_repository: + name: "{{ item.name }}" + description: "{{ item.description }}" + file: "{{ item.file }}" + baseurl: "{{ item.baseurl }}" + gpgcheck: "{{ item.gpgcheck | default('no') }}" + enabled: "{{ item.enabled | default('yes') }}" + repo_gpgcheck: "{{ item.repo_gpgcheck | default('no') }}" + gpgkey: "{{ item.gpgkey | default(omit) }}" + loop: "{{ wal_g_rpm_repository }}" + when: wal_g_install_repo + + - name: "Install WAL-G {{ wal_g_version }}" + dnf: + name: "{{ wal_g_package }}" + state: present + update_cache: yes + allow_downgrade: yes + tags: wal-g, wal_g, wal_g_install + + - name: "Install curl and jq" + dnf: + name: + - curl + - jq + state: present + update_cache: yes + +- name: Debian based distros + when: ansible_os_family == "Debian" + block: + - name: Install apt-transport-https + apt: + name: apt-transport-https + state: present + + - name: Add WAL-G repository + ansible.builtin.apt_repository: + repo: "{{ item.repo }}" + state: present + filename: "{{ item.filename }}" + update_cache: yes + loop: "{{ wal_g_apt_repository }}" + when: wal_g_install_repo + + - name: "Install WAL-G {{ wal_g_version }} and deps" + apt: + name: + - "{{ wal_g_package }}" + - libsodium23 + - liblzo2-2 + state: present + update_cache: yes + allow_downgrade: yes + tags: wal-g, wal_g, wal_g_install + + - name: "Install curl and jq" + apt: + name: + - curl + - jq + state: present + update_cache: yes + +- name: "Gather packages" + package_facts: + manager: auto + +- name: "Set fact about PostgreSQL package" + set_fact: + wal_g_postgresql_package: "{{ ansible_facts.packages | list | select('match', wal_g_postgresql_package_name_regex) | first }}" + +- name: "Get PostgreSQL major version" + set_fact: + wal_g_pg_major_version: "{{ ansible_facts.packages[wal_g_postgresql_package][0]['version'] | split('.') | first }}" + +- name: Create wal-g config + when: wal_g_config is defined + block: + - name: Merge user options for WAL-G part of config + set_fact: + wal_g_combined_config: "{{ wal_g_default_config | combine(wal_g_config|default({}), recursive=true) }}" + + - name: "Generate conf file walg.json" + copy: + content: "{{ wal_g_combined_config | to_nice_json }}" + dest: "{{ wal_g_pg_home_dir }}/{{ wal_g_config_name }}" + owner: "{{ wal_g_pg_config_owner }}" + group: "{{ wal_g_pg_config_group }}" + mode: 0644 + tags: wal-g, wal_g, wal_g_conf + +- name: Create wal-g restore config + when: wal_g_restore_config is defined + block: + - name: Merge user options for WAL-G part of restore config + set_fact: + wal_g_restore_combined_config: "{{ wal_g_default_restore_config | combine(wal_g_restore_config|default({}), recursive=true) }}" + + - name: "Generate conf file walg-restore.json" + copy: + content: "{{ wal_g_restore_combined_config | to_nice_json }}" + dest: "{{ wal_g_pg_home_dir }}/{{ wal_g_restore_config_name }}" + owner: "{{ wal_g_pg_config_owner }}" + group: "{{ wal_g_pg_config_group }}" + mode: 0644 diff --git a/templates/backup-permanent.service.j2 b/templates/backup-permanent.service.j2 new file mode 100644 index 0000000..00678c2 --- /dev/null +++ b/templates/backup-permanent.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Permanent backup PostgreSQL + +[Service] +Type=oneshot +ExecStart=/bin/bash {{ wal_g_pg_home_dir }}/backup-permanent.sh +User={{ wal_g_pg_backup_owner }} +Group={{ wal_g_pg_backup_group }} + +[Install] +WantedBy=multi-user.target diff --git a/templates/backup-permanent.sh.j2 b/templates/backup-permanent.sh.j2 new file mode 100644 index 0000000..fec1948 --- /dev/null +++ b/templates/backup-permanent.sh.j2 @@ -0,0 +1,13 @@ +exec 200>/tmp/backup.lock +flock -x 200 + +role=$(curl -k -s http://127.0.0.1:8008 || curl -k -s https://127.0.0.1:8008 | jq '.role') + + +if [ "${role}" == "\"master\"" ] || [ "${role}" == "" ]; then + {% if wal_g_http_proxy is defined %}http_proxy={{ wal_g_http_proxy }} {% else %}{% endif %}{% if wal_g_https_proxy is defined %}https_proxy={{ wal_g_https_proxy }} {% elif wal_g_http_proxy is defined %}https_proxy={{ wal_g_http_proxy }}{% else %}{% endif %} {{ wal_g_pg_binary_name }} --config {{ wal_g_pg_home_dir }}/{{ wal_g_config_name }} backup-push {{ wal_g_pg_data_dir }} --full --permanent 2>&1 | tee -a /var/log/postgresql/backup-permanent.log +else + echo "" +fi + +exec 200>&- diff --git a/templates/backup-permanent.timer.j2 b/templates/backup-permanent.timer.j2 new file mode 100644 index 0000000..5ac4502 --- /dev/null +++ b/templates/backup-permanent.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Run PostgreSQL backup permanent + +[Timer] +Unit=backup-permanent.service +OnCalendar={{ wal_g_backup_permanent_schedule }} + +[Install] +WantedBy=timers.target diff --git a/templates/backup-retention.service.j2 b/templates/backup-retention.service.j2 new file mode 100644 index 0000000..7504732 --- /dev/null +++ b/templates/backup-retention.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Backups retention for PostgreSQL +Wants=backup-retention.timer + +[Service] +Type=oneshot +ExecStart=/bin/bash {{ wal_g_pg_home_dir }}/backup-retention.sh +User={{ wal_g_pg_backup_owner }} +Group={{ wal_g_pg_backup_group }} + +[Install] +WantedBy=multi-user.target diff --git a/templates/backup-retention.sh.j2 b/templates/backup-retention.sh.j2 new file mode 100644 index 0000000..f113250 --- /dev/null +++ b/templates/backup-retention.sh.j2 @@ -0,0 +1,12 @@ +exec 200>/tmp/backup.lock +flock -x 200 + +role=$(curl -k -s http://127.0.0.1:8008 || curl -k -s https://127.0.0.1:8008 | jq '.role') + +if [ "${role}" == "\"master\"" ] || [ "${role}" == "" ]; then + {% if wal_g_http_proxy is defined %}http_proxy={{ wal_g_http_proxy }} {% else %}{% endif %}{% if wal_g_https_proxy is defined %}https_proxy={{ wal_g_https_proxy }} {% elif wal_g_http_proxy is defined %}https_proxy={{ wal_g_http_proxy }}{% else %}{% endif %} {{ wal_g_pg_binary_name }} --config {{ wal_g_pg_home_dir }}/{{ wal_g_config_name }} delete retain {{ wal_g_backup_retention_number }} --confirm --use-sentinel-time 2>&1 | tee -a /var/log/postgresql/backup-retention.log +else + echo "" +fi + +exec 200>&- diff --git a/templates/backup-retention.timer.j2 b/templates/backup-retention.timer.j2 new file mode 100644 index 0000000..fcf5014 --- /dev/null +++ b/templates/backup-retention.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Run PostgreSQL backup retention + +[Timer] +Unit=backup-retention.service +OnCalendar={{ wal_g_backup_retention_schedule }} + +[Install] +WantedBy=timers.target diff --git a/templates/backup.service.j2 b/templates/backup.service.j2 new file mode 100644 index 0000000..ebfe390 --- /dev/null +++ b/templates/backup.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Backup PostgreSQL +Wants=backup.timer + +[Service] +Type=oneshot +ExecStart=/bin/bash {{ wal_g_pg_home_dir }}/backup.sh +User={{ wal_g_pg_backup_owner }} +Group={{ wal_g_pg_backup_group }} + +[Install] +WantedBy=multi-user.target diff --git a/templates/backup.sh.j2 b/templates/backup.sh.j2 new file mode 100644 index 0000000..617f5f3 --- /dev/null +++ b/templates/backup.sh.j2 @@ -0,0 +1,12 @@ +exec 200>/tmp/backup.lock +flock -x 200 + +role=$(curl -k -s http://127.0.0.1:8008 || curl -k -s https://127.0.0.1:8008 | jq '.role') + +if [ "${role}" == "\"master\"" ] || [ "${role}" == "" ]; then + {% if wal_g_http_proxy is defined %}http_proxy={{ wal_g_http_proxy }} {% else %}{% endif %}{% if wal_g_https_proxy is defined %}https_proxy={{ wal_g_https_proxy }} {% elif wal_g_http_proxy is defined %}https_proxy={{ wal_g_http_proxy }}{% else %}{% endif %} {{ wal_g_pg_binary_name }} --config {{ wal_g_pg_home_dir }}/{{ wal_g_config_name }} backup-push {{ wal_g_pg_data_dir }} --full 2>&1 | tee -a /var/log/postgresql/backup.log +else + echo "" +fi + +exec 200>&- diff --git a/templates/backup.timer.j2 b/templates/backup.timer.j2 new file mode 100644 index 0000000..6da0972 --- /dev/null +++ b/templates/backup.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Run PostgreSQL backup + +[Timer] +Unit=backup.service +OnCalendar={{ wal_g_backup_schedule }} + +[Install] +WantedBy=timers.target diff --git a/templates/walg.json.j2 b/templates/walg.json.j2 new file mode 100644 index 0000000..53e9a39 --- /dev/null +++ b/templates/walg.json.j2 @@ -0,0 +1,7 @@ +{ +{% for wal_g in wal_g_json %} + "{{ wal_g.option }}": "{{ wal_g.value }}"{% if not loop.last %}, +{% endif %} +{% endfor %} + +} diff --git a/vars/Debian.yaml b/vars/Debian.yaml new file mode 100644 index 0000000..706080a --- /dev/null +++ b/vars/Debian.yaml @@ -0,0 +1,13 @@ +wal_g_apt_repository: + - repo: deb [trusted=yes] https://mirror.0xace.cc/debian/custom/ default all + filename: mirror.0xace.cc + +wal_g_pg_home_dir: "/var/lib/postgresql" +wal_g_pg_data_dir: "{{ wal_g_pg_home_dir }}/{{ wal_g_pg_major_version }}/{{ wal_g_pg_cluster_name }}" +wal_g_pg_cluster_name: "main" + +wal_g_postgresql_package_name_regex: 'postgresql-.[{{ wal_g_postgresql_supported_versions | join(",") }}]' + +wal_g_package_name: "wal-g" +wal_g_package: "{{ wal_g_package_name }}={{ wal_g_version }}-{{ wal_g_version_build }}" +wal_g_version_build: "1" diff --git a/vars/RedHat.yaml b/vars/RedHat.yaml new file mode 100644 index 0000000..51b3163 --- /dev/null +++ b/vars/RedHat.yaml @@ -0,0 +1,14 @@ +wal_g_rpm_repository: + - name: "mirror.0xace.cc" + file: "mirror.0xace.cc" + description: "0xace.cc repository" + baseurl: "https://mirror.0xace.cc/rhel/$releasever/custom/" + gpgcheck: no + +wal_g_pg_home_dir: "/var/lib/pgsql" +wal_g_pg_data_dir: "{{ wal_g_pg_home_dir }}/{{ wal_g_pg_major_version }}/data" + +wal_g_postgresql_package_name_regex: 'postgresql.[{{ wal_g_postgresql_supported_versions | join(",") }}]-server' + +wal_g_package_name: "wal-g" +wal_g_package: "{{ wal_g_package_name }}-{{ wal_g_version }}" diff --git a/vars/main.yaml b/vars/main.yaml new file mode 100644 index 0000000..e69de29