From 7e1c5cf9b5432a348e49df352708a766c514dc13 Mon Sep 17 00:00:00 2001 From: Emmanuel Cohen <emmanuel@hexack.fr> Date: Wed, 31 Mar 2021 09:07:42 +0000 Subject: [PATCH] double failover test using postgres-maintenance role, refs #33389 --- .gitlab-ci.yml | 4 +- Makefile | 9 +- .../tests/test_postgres_e_shutdown_primary.py | 21 ----- ...t_postgres_f_new_primary_cluster_status.py | 22 ----- .../test_postgres_g_new_primary_write.py | 12 --- .../test_postgres_h_reintegrate_server.py | 46 ----------- .../test_postgres_i_final_cluster_status.py | 22 ----- .../{ha-pgsql => pgsql-ha}/converge.yml | 0 .../{ha-pgsql => pgsql-ha}/molecule.yml | 2 + ansible/molecule/pgsql-ha/side_effect.yml | 13 +++ .../pgsql-ha/side_effect_destroy_master.yml | 53 ++++++++++++ .../{ha-pgsql => pgsql-ha}/tests/commons.py | 0 .../tests/test_postgres_a_setup.py | 0 .../tests/test_postgres_b_cluster_status.py | 0 ...st_postgres_c_test_cluster_db0_primary.py} | 0 ..._postgres_d_test_cluster_db1_secondary.py} | 0 ansible/playbooks/postgres-maintenance.yml | 82 ++----------------- .../fenced_to_standby.yml | 35 ++++++++ .../rephacheck_status.yml | 14 ++++ .../postgres-maintenance/restart_repmgrd.yml | 14 ++++ .../standby_to_primary.yml | 24 ++++++ 21 files changed, 171 insertions(+), 202 deletions(-) delete mode 100644 ansible/molecule/ha-pgsql/tests/test_postgres_e_shutdown_primary.py delete mode 100644 ansible/molecule/ha-pgsql/tests/test_postgres_f_new_primary_cluster_status.py delete mode 100644 ansible/molecule/ha-pgsql/tests/test_postgres_g_new_primary_write.py delete mode 100644 ansible/molecule/ha-pgsql/tests/test_postgres_h_reintegrate_server.py delete mode 100644 ansible/molecule/ha-pgsql/tests/test_postgres_i_final_cluster_status.py rename ansible/molecule/{ha-pgsql => pgsql-ha}/converge.yml (100%) rename ansible/molecule/{ha-pgsql => pgsql-ha}/molecule.yml (96%) create mode 100644 ansible/molecule/pgsql-ha/side_effect.yml create mode 100644 ansible/molecule/pgsql-ha/side_effect_destroy_master.yml rename ansible/molecule/{ha-pgsql => pgsql-ha}/tests/commons.py (100%) rename ansible/molecule/{ha-pgsql => pgsql-ha}/tests/test_postgres_a_setup.py (100%) rename ansible/molecule/{ha-pgsql => pgsql-ha}/tests/test_postgres_b_cluster_status.py (100%) rename ansible/molecule/{ha-pgsql/tests/test_postgres_c_test_cluster_primary.py => pgsql-ha/tests/test_postgres_c_test_cluster_db0_primary.py} (100%) rename ansible/molecule/{ha-pgsql/tests/test_postgres_d_test_cluster_secondary.py => pgsql-ha/tests/test_postgres_d_test_cluster_db1_secondary.py} (100%) create mode 100644 ansible/playbooks/postgres-maintenance/fenced_to_standby.yml create mode 100644 ansible/playbooks/postgres-maintenance/rephacheck_status.yml create mode 100644 ansible/playbooks/postgres-maintenance/restart_repmgrd.yml create mode 100644 ansible/playbooks/postgres-maintenance/standby_to_primary.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8dd3aa96..0d227345 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,7 +103,7 @@ stop-ovh-preprod: script: - make stop-preprod-ovh -test-ha-pgsql: +test-pgsql-ha: image: registry.ubicast.net/mediaserver/envsetup:root stage: test-pgsql-ha tags: @@ -115,7 +115,7 @@ test-ha-pgsql: changes: - ansible/**/* script: - - make test ha-pgsql=1 + - make test pgsql-ha=1 test-mediaserver-ha: image: registry.ubicast.net/mediaserver/envsetup:root diff --git a/Makefile b/Makefile index 29c61748..e32e7bb7 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,11 @@ endif ifdef keep MOLECULE_TEST_FLAGS += --destroy=never --parallel endif -ifdef ha-pgsql - MOLECULE_TEST_FLAGS += --scenario-name ha-pgsql +ifdef pgsql-ha + MOLECULE_TEST_FLAGS += --scenario-name pgsql-ha +endif +ifdef mediaserver-ha + MOLECULE_TEST_FLAGS += --scenario-name mediaserver-ha endif ifdef mediaserver-ha MOLECULE_TEST_FLAGS += --scenario-name mediaserver-ha @@ -65,7 +68,7 @@ lint: ANSIBLE_CONFIG=$(ANSIBLE_CONFIG) $(ANSIBLE_LINT_BIN) ansible/playbooks/site.yml .PHONY: test -## test: Run development tests on the project : debug=1, keep=1, SKYREACH_SYSTEM_KEY=<xxx>, ha-pgsql=1 +## test: Run development tests on the project : debug=1, keep=1, SKYREACH_SYSTEM_KEY=<xxx>, pgsql-ha=1 test: cd ansible; molecule $(MOLECULE_FLAGS) test $(MOLECULE_TEST_FLAGS) diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_e_shutdown_primary.py b/ansible/molecule/ha-pgsql/tests/test_postgres_e_shutdown_primary.py deleted file mode 100644 index 919ea2dd..00000000 --- a/ansible/molecule/ha-pgsql/tests/test_postgres_e_shutdown_primary.py +++ /dev/null @@ -1,21 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -import time - - -hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres") -testinfra_hosts = [i for i in hosts if i.startswith('db0')] - - -def test_postgresql_check_shutdown_primary(host): - ''' Shutdown the primary server ''' - - s = host.ansible("command", "systemctl stop postgresql", become=True, check=False) - assert s['changed'] - - time.sleep(40) - - s = host.socket("tcp://127.0.0.1:5432") - assert not s.is_listening diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_f_new_primary_cluster_status.py b/ansible/molecule/ha-pgsql/tests/test_postgres_f_new_primary_cluster_status.py deleted file mode 100644 index 9f9c8edb..00000000 --- a/ansible/molecule/ha-pgsql/tests/test_postgres_f_new_primary_cluster_status.py +++ /dev/null @@ -1,22 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -import commons - -# /!\ This test run accross all servers -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres") - - -def test_postgresql_check_repmgr_new_master(host): - ''' check repmgr status for each node after new master election ''' - - if host.ansible.get_variables()["inventory_hostname"].startswith("db0-default"): - data = commons.get_status(host) - assert data == "fenced" - if host.ansible.get_variables()["inventory_hostname"].startswith("db1-default"): - data = commons.get_status(host) - assert data == "primary" - if host.ansible.get_variables()["inventory_hostname"].startswith("db2-default"): - data = commons.get_status(host) - assert data == "witness" diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_g_new_primary_write.py b/ansible/molecule/ha-pgsql/tests/test_postgres_g_new_primary_write.py deleted file mode 100644 index 4aa79ed3..00000000 --- a/ansible/molecule/ha-pgsql/tests/test_postgres_g_new_primary_write.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - - -hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres") -testinfra_hosts = [i for i in hosts if i.startswith('db1')] - - -def test_postgresql_insert_new_master(host): - s = host.ansible("postgresql_query", "db=test query='INSERT INTO test_ha (name) VALUES (\'test2\');'", become=True, check=False, become_user='postgres') - assert s["changed"] diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_h_reintegrate_server.py b/ansible/molecule/ha-pgsql/tests/test_postgres_h_reintegrate_server.py deleted file mode 100644 index 17e3b486..00000000 --- a/ansible/molecule/ha-pgsql/tests/test_postgres_h_reintegrate_server.py +++ /dev/null @@ -1,46 +0,0 @@ -import os - -import testinfra.utils.ansible_runner -import time - -hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres") - - -testinfra_hosts = [i for i in hosts if i.startswith('db0')] - - -def test_postgresql_delete_data(host): - ''' delete data directory ''' - - s = host.ansible("command", "rm -rf /var/lib/postgresql/11/main/", become=True, check=False) - assert s['changed'] - - -def test_postgresql_launch_repmgr_sync(host): - ''' sync data with primary server using repmgr ''' - - current_m = [i for i in hosts if i.startswith('db1')] - #Â print("current master" + current_m[0]) - current_master = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_host(current_m[0]) - #Â current_master = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_host("db1-default") - #Â print(current_master) - current_master_ip = current_master.interface('eth0').addresses[0] - rep_mgr_command = "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby clone -h " + str(current_master_ip) + " -d repmgr -U repmgr -c" - s = host.ansible("command", rep_mgr_command, become=True, become_user='postgres', check=False) - assert s['changed'] - - -def test_postgresql_start_postgresql(host): - ''' start postgresql ''' - - s = host.ansible("command", "systemctl start postgresql", become=True, check=False) - time.sleep(20) - assert s['changed'] - - -def test_pogresql_register_as_standby(host): - ''' register server as standby in repmgr ''' - - s = host.ansible("command", "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby register", become=True, become_user='postgres', check=False) - time.sleep(20) - assert s['changed'] diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_i_final_cluster_status.py b/ansible/molecule/ha-pgsql/tests/test_postgres_i_final_cluster_status.py deleted file mode 100644 index 558b4028..00000000 --- a/ansible/molecule/ha-pgsql/tests/test_postgres_i_final_cluster_status.py +++ /dev/null @@ -1,22 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -import commons - -# /!\ This test run accross all servers -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres") - - -def test_postgresql_check_status_after_shutdown(host): - ''' check repmgr status accross server after primary change and server reintegration ''' - - if host.ansible.get_variables()["inventory_hostname"].startswith("db0"): - data = commons.get_status(host) - assert data == "standby" - if host.ansible.get_variables()["inventory_hostname"].startswith("db1"): - data = commons.get_status(host) - assert data == "primary" - if host.ansible.get_variables()["inventory_hostname"].startswith("db2"): - data = commons.get_status(host) - assert data == "witness" diff --git a/ansible/molecule/ha-pgsql/converge.yml b/ansible/molecule/pgsql-ha/converge.yml similarity index 100% rename from ansible/molecule/ha-pgsql/converge.yml rename to ansible/molecule/pgsql-ha/converge.yml diff --git a/ansible/molecule/ha-pgsql/molecule.yml b/ansible/molecule/pgsql-ha/molecule.yml similarity index 96% rename from ansible/molecule/ha-pgsql/molecule.yml rename to ansible/molecule/pgsql-ha/molecule.yml index 06ab89f3..5fd1e88a 100644 --- a/ansible/molecule/ha-pgsql/molecule.yml +++ b/ansible/molecule/pgsql-ha/molecule.yml @@ -49,6 +49,8 @@ provisioner: ANSIBLE_ACTION_PLUGINS: ../../plugins/action ANSIBLE_PYTHON_INTERPRETER: /usr/bin/python3 SKYREACH_SYSTEM_KEY: s1121eb6e7593525bf3e0302586c82d2 + playbooks: + side_effect: side_effect.yml verifier: name: testinfra #Â options: diff --git a/ansible/molecule/pgsql-ha/side_effect.yml b/ansible/molecule/pgsql-ha/side_effect.yml new file mode 100644 index 00000000..b66b60b2 --- /dev/null +++ b/ansible/molecule/pgsql-ha/side_effect.yml @@ -0,0 +1,13 @@ +--- +- name: first failover + import_playbook: side_effect_destroy_master.yml + +# Refresh inventory to have a clean inventory (each server in the initial group) +- name: refresh inventory + hosts: all + tasks: + - name: refresh inventory + meta: refresh_inventory + +- name: second failover + import_playbook: side_effect_destroy_master.yml diff --git a/ansible/molecule/pgsql-ha/side_effect_destroy_master.yml b/ansible/molecule/pgsql-ha/side_effect_destroy_master.yml new file mode 100644 index 00000000..5332953f --- /dev/null +++ b/ansible/molecule/pgsql-ha/side_effect_destroy_master.yml @@ -0,0 +1,53 @@ +--- +- name: stop master node + hosts: postgres[0] + tasks: + - name: stop postgresql + service: + name: postgresql@11-main + state: stopped + +- name: pause for 45 seconds + hosts: localhost + tasks: + - pause: + seconds: 45 + +- name: set correct host to correct groups + hosts: postgres + tasks: + - name: add db0 to fenced group + add_host: + name: "{{ groups['postgres'][0] }}" + groups: [ 'postgres_fenced', 'postgres_standby' ] + + - name: add db1 to primary group + add_host: + name: "{{ groups['postgres'][1] }}" + groups: postgres_primary + +# db0 is now fenced, import playbook to switch it back to standby +- import_playbook: ../../playbooks/postgres-maintenance/rephacheck_status.yml +- import_playbook: ../../playbooks/postgres-maintenance/fenced_to_standby.yml + +- name: pause for 45 seconds + hosts: localhost + tasks: + - pause: + seconds: 45 + +# db0 is now standby, import playbook to switch it back as primary +- import_playbook: ../../playbooks/postgres-maintenance/rephacheck_status.yml +- import_playbook: ../../playbooks/postgres-maintenance/standby_to_primary.yml + +- name: pause for 45 seconds + hosts: localhost + tasks: + - pause: + seconds: 45 + +# db0 is now primary, restart repmgrd +- import_playbook: ../../playbooks/postgres-maintenance/rephacheck_status.yml +- import_playbook: ../../playbooks/postgres-maintenance/restart_repmgrd.yml + +... diff --git a/ansible/molecule/ha-pgsql/tests/commons.py b/ansible/molecule/pgsql-ha/tests/commons.py similarity index 100% rename from ansible/molecule/ha-pgsql/tests/commons.py rename to ansible/molecule/pgsql-ha/tests/commons.py diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_a_setup.py b/ansible/molecule/pgsql-ha/tests/test_postgres_a_setup.py similarity index 100% rename from ansible/molecule/ha-pgsql/tests/test_postgres_a_setup.py rename to ansible/molecule/pgsql-ha/tests/test_postgres_a_setup.py diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_b_cluster_status.py b/ansible/molecule/pgsql-ha/tests/test_postgres_b_cluster_status.py similarity index 100% rename from ansible/molecule/ha-pgsql/tests/test_postgres_b_cluster_status.py rename to ansible/molecule/pgsql-ha/tests/test_postgres_b_cluster_status.py diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_c_test_cluster_primary.py b/ansible/molecule/pgsql-ha/tests/test_postgres_c_test_cluster_db0_primary.py similarity index 100% rename from ansible/molecule/ha-pgsql/tests/test_postgres_c_test_cluster_primary.py rename to ansible/molecule/pgsql-ha/tests/test_postgres_c_test_cluster_db0_primary.py diff --git a/ansible/molecule/ha-pgsql/tests/test_postgres_d_test_cluster_secondary.py b/ansible/molecule/pgsql-ha/tests/test_postgres_d_test_cluster_db1_secondary.py similarity index 100% rename from ansible/molecule/ha-pgsql/tests/test_postgres_d_test_cluster_secondary.py rename to ansible/molecule/pgsql-ha/tests/test_postgres_d_test_cluster_db1_secondary.py diff --git a/ansible/playbooks/postgres-maintenance.yml b/ansible/playbooks/postgres-maintenance.yml index 4385d878..b4774ee9 100755 --- a/ansible/playbooks/postgres-maintenance.yml +++ b/ansible/playbooks/postgres-maintenance.yml @@ -1,82 +1,16 @@ #!/usr/bin/env ansible-playbook --- -- name: GATHER FACTS - hosts: postgres_primary:postgres_standby:postgres_fenced - tags: always - tasks: - - name: get cluster state - command: "rephacheck" - register: rephacheck - - name: show status for each node - debug: - msg: "Current node {{ ansible_hostname }} status {{ rephacheck['stdout'] }}" - when: rephacheck['stdout'] != "" +- import_playbook: postgres-maintenance/rephacheck_status.yml + tags: [ 'always' ] -- name: POSTGRESQL SWITCH CURRENT STANDBY TO PRIMARY - hosts: postgres_standby - tags: [ 'never', 'standby-to-primary' ] - tasks: - - name: fail if node status if not standby - fail: - msg: "Current status {{ rephacheck['stdout'] }} must be standby." - when: rephacheck['stdout'] != "standby" - - name: check if node is currently in standby - command: "repmgr standby switchover -f /etc/postgresql/11/main/repmgr.conf --siblings-follow --dry-run" - become: true - become_user: postgres - when: rephacheck['stdout'] == "standby" - register: standby_dry_run - - name: switch standby node to primary - command: "repmgr standby switchover -f /etc/postgresql/11/main/repmgr.conf --siblings-follow" - become: true - become_user: postgres - when: - - standby_dry_run is succeeded - - rephacheck['stdout'] == "standby" - -- name: POSTGRESQL SWITCH CURRENT FENCED TO STANDBY - hosts: postgres_fenced +- import_playbook: postgres-maintenance/fenced_to_standby.yml tags: [ 'never', 'fenced-to-standby' ] - tasks: - - name: fail if node status if not fenced - fail: - msg: "Current status {{ rephacheck['stdout'] }} must be fenced." - when: rephacheck['stdout'] != "fenced" - - name: stop postgresql - systemd: - name: postgresql - state: stopped - - name: delete postgresql data directory - file: - path: /var/lib/postgresql/11/main/ - state: absent - force: true - - name: copy data from primary - command: "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby clone -h {{ hostvars[groups['postgres_primary'][0]]['ansible_default_ipv4']['address'] }} -d repmgr -U repmgr -c" - become: true - become_user: postgres - register: copy_from_primary - - name: start postgresql - systemd: - name: postgresql - state: started - when: copy_from_primary is succeeded - - name: register node as standby - command: "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby register" - become: true - become_user: postgres - when: copy_from_primary is succeeded -- name: REPMGRD RESTART - hosts: postgres +- import_playbook: postgres-maintenance/standby_to_primary.yml + tags: [ 'never', 'standby-to-primary' ] + +- import_playbook: postgres-maintenance/restart_repmgrd.yml tags: [ 'never', 'restart-repmgrd', 'standby-to-primary' ] - tasks: - - name: kill repmgrd - command: "pkill repmgrd" - ignore_errors: true - - name: restart repmgrd - systemd: - name: repmgrd - state: restarted + ... diff --git a/ansible/playbooks/postgres-maintenance/fenced_to_standby.yml b/ansible/playbooks/postgres-maintenance/fenced_to_standby.yml new file mode 100644 index 00000000..f4b19d1a --- /dev/null +++ b/ansible/playbooks/postgres-maintenance/fenced_to_standby.yml @@ -0,0 +1,35 @@ +#!/usr/bin/env ansible-playbook +--- +- name: POSTGRESQL SWITCH CURRENT FENCED TO STANDBY + hosts: postgres_fenced + tasks: + - name: fail if node status if not fenced + fail: + msg: "Current status {{ rephacheck['stdout'] }} must be fenced." + when: rephacheck['stdout'] != "fenced" + - name: stop postgresql + systemd: + name: postgresql + state: stopped + - name: delete postgresql data directory + file: + path: /var/lib/postgresql/11/main/ + state: absent + force: true + - name: copy data from primary + command: "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby clone -h {{ hostvars[groups['postgres_primary'][0]]['ansible_default_ipv4']['address'] }} -d repmgr -U repmgr -c" + become: true + become_user: postgres + register: copy_from_primary + - name: start postgresql + systemd: + name: postgresql + state: started + when: copy_from_primary is succeeded + - name: register node as standby + command: "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby register" + become: true + become_user: postgres + when: copy_from_primary is succeeded + +... diff --git a/ansible/playbooks/postgres-maintenance/rephacheck_status.yml b/ansible/playbooks/postgres-maintenance/rephacheck_status.yml new file mode 100644 index 00000000..b22dec19 --- /dev/null +++ b/ansible/playbooks/postgres-maintenance/rephacheck_status.yml @@ -0,0 +1,14 @@ +#!/usr/bin/env ansible-playbook +--- +- name: REHACHECK STATUS + hosts: postgres_primary:postgres_standby:postgres_fenced + tasks: + - name: get cluster state + command: "rephacheck" + register: rephacheck + - name: show status for each node + debug: + msg: "Current node {{ ansible_hostname }} status {{ rephacheck['stdout'] }}" + when: rephacheck['stdout'] != "" + +... diff --git a/ansible/playbooks/postgres-maintenance/restart_repmgrd.yml b/ansible/playbooks/postgres-maintenance/restart_repmgrd.yml new file mode 100644 index 00000000..4753a56a --- /dev/null +++ b/ansible/playbooks/postgres-maintenance/restart_repmgrd.yml @@ -0,0 +1,14 @@ +#!/usr/bin/env ansible-playbook +--- +- name: REPMGRD RESTART + hosts: postgres + tasks: + - name: kill repmgrd + command: "pkill repmgrd" + ignore_errors: true + - name: restart repmgrd + systemd: + name: repmgrd + state: restarted + +... diff --git a/ansible/playbooks/postgres-maintenance/standby_to_primary.yml b/ansible/playbooks/postgres-maintenance/standby_to_primary.yml new file mode 100644 index 00000000..bfce1c64 --- /dev/null +++ b/ansible/playbooks/postgres-maintenance/standby_to_primary.yml @@ -0,0 +1,24 @@ +#!/usr/bin/env ansible-playbook +--- +- name: POSTGRESQL SWITCH CURRENT STANDBY TO PRIMARY + hosts: postgres_standby + tasks: + - name: fail if node status if not standby + fail: + msg: "Current status {{ rephacheck['stdout'] }} must be standby." + when: rephacheck['stdout'] != "standby" + - name: check if node is currently in standby + command: "repmgr standby switchover -f /etc/postgresql/11/main/repmgr.conf --siblings-follow --dry-run" + become: true + become_user: postgres + when: rephacheck['stdout'] == "standby" + register: standby_dry_run + - name: switch standby node to primary + command: "repmgr standby switchover -f /etc/postgresql/11/main/repmgr.conf --siblings-follow" + become: true + become_user: postgres + when: + - standby_dry_run is succeeded + - rephacheck['stdout'] == "standby" + +... -- GitLab