diff --git a/defaults/main.yaml b/defaults/main.yaml index 5e8d988..3081210 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -23,7 +23,8 @@ patroni_replication_password: "replicator" # Patroni REST API options patroni_restapi_username: "admin" patroni_restapi_password: "admin" -patroni_restapi_listen_port: 8008 +patroni_restapi_listen_port: "8008" +patroni_restapi_listen_address: "0.0.0.0" # Patroni SSL options patroni_ssl: false @@ -36,21 +37,30 @@ patroni_play_group: "patroni" patroni_dcs_exists: false patroni_dcs_type: "consul" -## Backup and restore options +## WAL-G backup and restore options patroni_wal_g_install: false +patroni_wal_g_restore_from_backup: false +patroni_wal_g_pitr: 'latest' +patroni_wal_g_restore_backup_name: 'LATEST' patroni_wal_g_pg_binary_name: 'wal-g-pg' -patroni_postgresql_archive_mode: "{{ 'on' if patroni_wal_g_install else 'off' }}" -patroni_postgresql_archive_command: "{{ 'http_proxy=' ~ wal_g_http_proxy ~ ' ' if wal_g_http_proxy is defined else '' }}{{ 'https_proxy=' ~ wal_g_https_proxy ~ ' ' if wal_g_https_proxy is defined else 'https_proxy=' ~ wal_g_http_proxy ~ ' ' if wal_g_http_proxy is defined else '' }}{{ wal_g_pg_binary_name ~ ' wal-push %p' if patroni_wal_g_install else '' }}" +patroni_wal_g_config_name: ".walg.json" +patroni_wal_g_restore_config_name: ".walg-restore.json" +patroni_restore_script: "{{ patroni_postgresql_home_dir }}/restore.sh" +#patroni_postgresql_archive_command: "wal-g wal-push %p" +#patroni_postgresql_restore_command: "wal-g wal-fetch %f %p" +#patroni_cluster_bootstrap_command: "wal-g backup-fetch latest" +## "restore_command" written to recovery.conf when configuring follower (create replica) +#patroni_cluster_restore_command: "wal-g wal-fetch %f %p" +#patroni_wal_g_create_replica_methods: +# - {option: "command", value: "{{ patroni_cluster_bootstrap_command }}"} +# - {option: "no_params", value: "True"} +#patroni_basebackup_create_replica_methods: +# - {option: "max-rate", value: "1000M"} +# - {option: "checkpoint", value: "fast"} -patroni_wal_g_create_replica_methods: - - {option: "command", value: "{{ 'http_proxy=' ~ wal_g_http_proxy ~ ' ' if wal_g_http_proxy is defined else '' }}{{ 'https_proxy=' ~ wal_g_https_proxy ~ ' ' if wal_g_https_proxy is defined else 'https_proxy=' ~ wal_g_http_proxy ~ ' ' if wal_g_http_proxy is defined else '' }}{{ wal_g_pg_binary_name }} backup-fetch {{ patroni_postgresql_data_dir }} LATEST"} - - {option: "no_params", value: "True"} -patroni_basebackup_create_replica_methods: - - {option: "max-rate", value: "1000M"} - - {option: "checkpoint", value: "fast"} - -# "restore_command" written to recovery.conf when configuring follower (create replica) -patroni_postgresql_restore_command: "{{ 'http_proxy=' ~ wal_g_http_proxy ~ ' ' if wal_g_http_proxy is defined else '' }}{{ 'https_proxy=' ~ wal_g_https_proxy ~ ' ' if wal_g_https_proxy is defined else 'https_proxy=' ~ wal_g_http_proxy ~ ' ' if wal_g_http_proxy is defined else '' }}{{ wal_g_pg_binary_name ~ ' wal-fetch %f %p' if patroni_wal_g_install else '' }}" +# More examples +# patroni_postgresql_restore_command: "pgbackrest --stanza={{ pgbackrest_stanza }} archive-get %f %p" # restore WAL-s using pgbackrest +# patroni_postgresql_restore_command: "pg_probackup-{{ pg_probackup_version }} archive-get -B {{ pg_probackup_dir }} --instance {{ pg_probackup_instance }} --wal-file-path=%p --wal-file-name=%f" # restore WAL-s using pg_probackup patroni_synchronous_mode: false # or 'true' for enable synchronous database replication patroni_synchronous_mode_strict: false # if 'true' then block all client writes to the master, when a synchronous replica is not available @@ -94,11 +104,11 @@ patroni_postgresql_backup_parameters: parameters: archive_command: "{{ patroni_postgresql_archive_command if patroni_wal_g_install else 'cd .' }}" restore_command: "{{ patroni_postgresql_restore_command if patroni_wal_g_install else 'cd .' }}" - archive_mode: "on" - archive_timeout: "1800s" # postgresql parameters to bootstrap dcs (are parameters for example) patroni_postgresql_default_parameters: + archive_mode: "on" + archive_timeout: "1800s" max_connections: "1000" superuser_reserved_connections: "5" password_encryption: "{{ patroni_postgresql_password_encryption_algorithm }}" diff --git a/tasks/config.yaml b/tasks/config.yaml index 757e4eb..56042de 100644 --- a/tasks/config.yaml +++ b/tasks/config.yaml @@ -3,10 +3,6 @@ set_fact: patroni_postgresql_parameters_combined: "{{ patroni_postgresql_default_parameters | combine(patroni_postgresql_parameters|default({}), recursive=true) }}" -- name: Merge backup options for PostgreSQL part of Patroni config - set_fact: - patroni_postgresql_parameters_combined: "{{ patroni_postgresql_parameters_combined | combine(patroni_postgresql_backup_parameters['postgresql']['parameters']|default({}), recursive=true) }}" - - name: Template Patroni default config set_fact: patroni_default_config: "{{ lookup('template', 'patroni.yaml.j2') | from_yaml }}" @@ -45,3 +41,11 @@ register: patroni_config_file when: not patroni_config_file_exists_result.stat.exists notify: Restart Patroni + +- name: Propagate Patroni restore script + template: + src: "restore.sh.j2" + dest: "{{ patroni_restore_script }}" + owner: "postgres" + group: "postgres" + mode: 0755 diff --git a/tasks/dynamic_config.yaml b/tasks/dynamic_config.yaml index ea9eb23..66211b5 100644 --- a/tasks/dynamic_config.yaml +++ b/tasks/dynamic_config.yaml @@ -15,7 +15,22 @@ - name: Dynamic configuration parameters for Patroni when: patroni_dynamic_config_file.changed block: - - name: Find Patroni master + - name: Wait for Patroni leader + vars: + _query: 'json.members[].role' + ansible.builtin.uri: + url: "{{ 'https://' if patroni_ssl else 'http://' }}{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}:{{ patroni_restapi_listen_port }}/cluster" + method: GET + body_format: json + status_code: + - 200 + - 503 + register: patroni_role + until: patroni_role | json_query(_query) | regex_findall('leader') | count > 0 + retries: 100 + delay: 10 + + - name: Get Patroni nodes roles ansible.builtin.uri: url: "{{ 'https://' if patroni_ssl else 'http://' }}{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}:{{ patroni_restapi_listen_port }}" method: GET @@ -39,4 +54,5 @@ force_basic_auth: yes when: - "hostvars[inventory_hostname]['role'] == 'master'" - + rescue: + - include_tasks: dynamic_config.yaml diff --git a/tasks/main.yaml b/tasks/main.yaml index e04de5f..e3834ae 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -70,9 +70,10 @@ register: patroni_enable_and_start tags: patroni, patroni_start -- name: "Dynamic Patroni configuration" - include_tasks: dynamic_config.yaml - #when: - # - patroni_postgresql_dynamic_parameters - tags: patroni_dynamic_configure +#- name: Patroni cluster reconfiguration after restore from backup +# include_tasks: restore.yaml +# when: patroni_wal_g_restore_from_backup +- name: Dynamic Patroni configuration + include_tasks: dynamic_config.yaml + tags: patroni_dynamic_configure diff --git a/tasks/restore.yaml b/tasks/restore.yaml new file mode 100644 index 0000000..b49b68f --- /dev/null +++ b/tasks/restore.yaml @@ -0,0 +1,7 @@ +--- +- name: Copy walg-backup to walg-restore after restore + copy: + src: "{{ patroni_postgresql_home_dir ~ '/' ~ patroni_wal_g_backup_config_name }}" + dest: "{{ patroni_postgresql_home_dir ~ '/' ~ patroni_wal_g_restore_config_name }}" + remote_src: true + when: patroni_wal_g_backup diff --git a/templates/patroni.yaml.j2 b/templates/patroni.yaml.j2 index d8c646d..f89aa13 100644 --- a/templates/patroni.yaml.j2 +++ b/templates/patroni.yaml.j2 @@ -25,7 +25,7 @@ ctl: cafile: {{ patroni_ssl_path }}/CA-cert.crt restapi: - listen: 0.0.0.0:{{ patroni_restapi_listen_port }} + listen: {{ patroni_restapi_listen_address }}:{{ patroni_restapi_listen_port }} connect_address: {{ hostvars[inventory_hostname]['ansible_host'] }}:{{ patroni_restapi_listen_port }} {% if patroni_ssl|bool %} certfile: {{ patroni_ssl_path }}/{{ patroni_self_signed_cert_name }}.crt @@ -57,12 +57,12 @@ bootstrap: method: {{ patroni_cluster_bootstrap_method }} {% if patroni_cluster_bootstrap_method == 'wal-g' %} wal-g: - command: {{ patroni_wal_g_cluster_bootstrap_command }} + command: {{ patroni_restore_script }} no_params: True recovery_conf: recovery_target_action: promote - recovery_target_timeline: latest - restore_command: {{ patroni_wal_g_pg_binary_name }} wal-fetch %f %p + recovery_target_timeline: {{ patroni_wal_g_pitr }} + restore_command: {{ patroni_cluster_restore_command }} {% endif %} {% if patroni_cluster_bootstrap_method == 'pg_probackup' %} pg_probackup: @@ -126,9 +126,9 @@ postgresql: listen: {{ hostvars[inventory_hostname]['ansible_host'] }},127.0.0.1:{{ patroni_postgresql_port }} connect_address: {{ hostvars[inventory_hostname]['ansible_host'] }}:{{ patroni_postgresql_port }} use_unix_socket: true - data_dir: {{ patroni_postgresql_home_dir }}/{{ patroni_postgresql_major_version }}/data + data_dir: {{ patroni_postgresql_home_dir }}/{{ patroni_postgresql_major_version }}/{{ patroni_postgresql_cluster_name }} bin_dir: {{ patroni_postgresql_bin_dir }} - config_dir: {{ patroni_postgresql_home_dir }}/{{ patroni_postgresql_major_version }}/data + config_dir: {{ patroni_postgresql_home_dir }}/{{ patroni_postgresql_major_version }}/{{ patroni_postgresql_cluster_name }} pgpass: {{ patroni_postgresql_home_dir }}/.pgpass_patroni authentication: replication: @@ -215,11 +215,6 @@ postgresql: max-rate: '100M' {% endif %} -{% if patroni_postgresql_restore_command is defined and patroni_postgresql_restore_command | length > 0 and patroni_wal_g_install %} - recovery_conf: - restore_command: {{ patroni_postgresql_restore_command }} -{% endif %} - watchdog: mode: off # Allowed values: off, automatic, required device: /dev/watchdog diff --git a/templates/restore.sh.j2 b/templates/restore.sh.j2 new file mode 100644 index 0000000..5c4c187 --- /dev/null +++ b/templates/restore.sh.j2 @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +{{ patroni_cluster_bootstrap_command }} diff --git a/vars/Debian-11.yaml b/vars/Debian-11.yaml index 3d39dad..27cd095 100644 --- a/vars/Debian-11.yaml +++ b/vars/Debian-11.yaml @@ -7,7 +7,7 @@ patroni_apt_repository: filename: postgresql patroni_deps_packages: - - gnupg + - gnupg2 - python3-psycopg2 - python3-consul - python3-zookeeper diff --git a/vars/Debian-12.yaml b/vars/Debian-12.yaml index 3d39dad..27cd095 100644 --- a/vars/Debian-12.yaml +++ b/vars/Debian-12.yaml @@ -7,7 +7,7 @@ patroni_apt_repository: filename: postgresql patroni_deps_packages: - - gnupg + - gnupg2 - python3-psycopg2 - python3-consul - python3-zookeeper diff --git a/vars/RedHat.yaml b/vars/RedHat.yaml index b02f155..479e58f 100644 --- a/vars/RedHat.yaml +++ b/vars/RedHat.yaml @@ -23,6 +23,7 @@ patroni_postgresql_package_name: "postgresql" patroni_postgresql_package: "{{ patroni_postgresql_package_name }}-{{ patroni_postgresql_version }}" patroni_postgresql_data_dir: "{{ patroni_postgresql_home_dir }}/{{ patroni_postgresql_major_version }}/data" patroni_postgresql_wal_dir: "" +patroni_postgresql_cluster_name: "data" patroni_postgresql_conf_dir: "{{ patroni_postgresql_data_dir }}" patroni_postgresql_log_dir: "/var/log/postgresql" patroni_postgresql_unix_socket_dir: "/var/run/postgresql" diff --git a/vars/Ubuntu-20.04.yaml b/vars/Ubuntu-20.04.yaml index 416adaf..df67c06 100644 --- a/vars/Ubuntu-20.04.yaml +++ b/vars/Ubuntu-20.04.yaml @@ -7,7 +7,7 @@ patroni_apt_repository: filename: postgresql patroni_deps_packages: - - gnupg + - gnupg2 - python3-psycopg2 - python3-consul - python3-zookeeper diff --git a/vars/Ubuntu-22.04.yaml b/vars/Ubuntu-22.04.yaml index 416adaf..df67c06 100644 --- a/vars/Ubuntu-22.04.yaml +++ b/vars/Ubuntu-22.04.yaml @@ -7,7 +7,7 @@ patroni_apt_repository: filename: postgresql patroni_deps_packages: - - gnupg + - gnupg2 - python3-psycopg2 - python3-consul - python3-zookeeper