Molecule tests ha pgsql (refs #32798) and docker deployment (refs #32859)

.gitlab-ci.yml

@@ -71,4 +71,18 @@ test:
   script:
     - make test
 
+test-ha-pgsql:
+  image: registry.ubicast.net/mediaserver/envsetup:root
+  stage: test
+  tags:
+    - docker
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_requests"'
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      changes:
+        - ansible/**/*
+  script:
+    - make test ha-pgsql=1
+
 ...

Makefile

@@ -18,6 +18,9 @@ endif
 ifdef keep
 	MOLECULE_TEST_FLAGS += --destroy=never --parallel
 endif
+ifdef ha-pgsql
+	MOLECULE_TEST_FLAGS += --scenario-name ha-pgsql
+endif
 
 .PHONY: all
 ## TARGET: DESCRIPTION: ARGS
@@ -58,7 +61,7 @@ lint:
 	ANSIBLE_CONFIG=$(ANSIBLE_CONFIG) $(ANSIBLE_LINT_BIN) ansible/playbooks/site.yml
 
 .PHONY: test
-## test: Run development tests on the project : debug=1, keep=1, SKYREACH_SYSTEM_KEY=<xxx>
+## test: Run development tests on the project : debug=1, keep=1, SKYREACH_SYSTEM_KEY=<xxx>, ha-pgsql=1
 test:
 ifndef SKYREACH_SYSTEM_KEY
 	$(error SKYREACH_SYSTEM_KEY is undefined)

ansible/molecule/default/molecule.yml

@@ -24,6 +24,8 @@ platforms:
       - netcapture
 provisioner:
   name: ansible
+  options:
+    D: true
   env:
     ANSIBLE_ROLES_PATH: ../../roles
     ANSIBLE_LIBRARY: ../../library

ansible/molecule/ha-pgsql/converge.yml

+#!/usr/bin/env ansible-playbook
+---
+
+- name: PYTHON
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: ensure python3 is installed
+      register: python_install
+      changed_when: "'es_pyinstall' in python_install.stdout_lines"
+      raw: command -v python3 || echo es_pyinstall && apt update && apt install -y python3-minimal python3-apt
+
+- name: Converge
+  hosts: postgres
+  pre_tasks:
+    - name: check running in a docker container
+      register: check_if_docker
+      stat:
+        path: /.dockerenv
+    - name: set docker flag variable
+      set_fact:
+        in_docker: "{{ check_if_docker.stat.exists | d(false) }}"
+  roles:
+    - base
+    - postgres-ha
+  post_tasks:
+    - name: deploy letsencrypt certificate
+      when: letsencrypt_enabled | d(false)
+      include_role:
+        name: letsencrypt
+    - name: configure network
+      when: network_apply | d(false)
+      include_role:
+        name: network
+    - name: configure proxy
+      when: proxy_apply | d(false)
+      include_role:
+        name: proxy
+
+...

ansible/molecule/ha-pgsql/molecule.yml

+---
+driver:
+  name: docker
+platforms:
+  - name: db0-default
+    image: registry.ubicast.net/docker/debian-systemd:buster
+    command: /lib/systemd/systemd
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+    groups:
+      - postgres
+  - name: db1-default
+    image: registry.ubicast.net/docker/debian-systemd:buster
+    command: /lib/systemd/systemd
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+    groups:
+      - postgres
+  - name: db2-default
+    image: registry.ubicast.net/docker/debian-systemd:buster
+    command: /lib/systemd/systemd
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    tmpfs:
+      - /tmp
+      - /run
+    groups:
+      - postgres
+provisioner:
+  name: ansible
+  options:
+    D: true
+  inventory:
+    group_vars:
+      postgres:
+        repmgr_password: "testrepmgr"
+  env:
+    ANSIBLE_ROLES_PATH: ../../roles
+    ANSIBLE_LIBRARY: ../../library
+    ANSIBLE_ACTION_PLUGINS: ../../plugins/action
+    ANSIBLE_PYTHON_INTERPRETER: /usr/bin/python3
+    SKYREACH_SYSTEM_KEY: s1121eb6e7593525bf3e0302586c82d2
+
+verifier:
+  name: testinfra
+  options:
+    verbose: true

ansible/molecule/ha-pgsql/tests/commons.py

+import socket
+
+
+def get_status(host):
+    ip = host.interface('eth0').addresses[0]
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.connect((ip, 8543))
+    data = s.recv(1024)
+    return data.rstrip().decode('utf-8')

ansible/molecule/ha-pgsql/tests/test_postgres_a_setup.py

+import os
+
+import testinfra.utils.ansible_runner
+
+# /!\ This test run accross all servers
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ["MOLECULE_INVENTORY_FILE"]
+).get_hosts("postgres")
+
+
+def test_psycopg2_is_installed(host):
+    p = host.package("python3-psycopg2")
+
+    assert p.is_installed
+
+
+def test_postgres_is_installed(host):
+    p = host.package("postgresql-11")
+
+    assert p.is_installed
+    assert p.version.startswith("11")
+
+
+def test_postgres_user(host):
+    u = host.user("postgres")
+
+    assert u.name == "postgres"
+
+
+def test_postgres_service(host):
+    s = host.service("postgresql@11-main")
+
+    assert s.is_running
+
+
+def test_postgresql_socket(host):
+    s = host.socket("tcp://127.0.0.1:5432")
+
+    assert s.is_listening

ansible/molecule/ha-pgsql/tests/test_postgres_b_cluster_status.py

+import os
+
+import testinfra.utils.ansible_runner
+
+import commons
+
+# This test run accross all servers
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres")
+
+
+def test_postgresql_check_repmgr_status(host):
+    ''' check if repmgr is working correctly on each node'''
+
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db0-default"):
+        data = commons.get_status(host)
+        assert data == "primary"
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db1-default"):
+        data = commons.get_status(host)
+        assert data == "standby"
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db2-default"):
+        data = commons.get_status(host)
+        assert data == "witness"

ansible/molecule/ha-pgsql/tests/test_postgres_c_test_cluster_primary.py

+import os
+
+import testinfra.utils.ansible_runner
+
+# This test run accross all servers
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("db0-default")
+
+
+def test_postgresql_create_db(host):
+    ''' check if we can only create db on the primary node of the cluster '''
+
+    s = host.ansible("postgresql_db", "name=test", become=True, check=False, become_user='postgres')
+    assert s["changed"]
+
+
+def test_postgresql_create_table(host):
+    ''' check if we can only create a table on the primary node of the cluster '''
+
+    s = host.ansible("postgresql_query", "db=test query='CREATE TABLE test_ha (id SERIAL PRIMARY KEY, name VARCHAR(100) );'", become=True, check=False, become_user='postgres')
+    assert s["changed"]
+
+
+def test_postgresql_insert(host):
+    ''' check if we can only write to the primary node of the cluster '''
+
+    s = host.ansible("postgresql_query", "db=test query='INSERT INTO test_ha (name) VALUES (\'test\');'", become=True, check=False, become_user='postgres')
+    assert s["changed"]

ansible/molecule/ha-pgsql/tests/test_postgres_d_test_cluster_secondary.py

+import os
+
+import testinfra.utils.ansible_runner
+
+# This test run accross all servers
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("db1-default")
+
+
+def test_postgresql_create_db(host):
+    ''' check if we can only create db on the primary node of the cluster '''
+
+    s = host.ansible("postgresql_db", "name=test", become=True, check=False, become_user='postgres')
+    assert not s["changed"]
+
+
+def test_postgresql_create_table(host):
+    ''' check if we can only create a table on the primary node of the cluster '''
+
+    s = host.ansible("postgresql_query", "db=test query='CREATE TABLE test_ha (id SERIAL PRIMARY KEY, name VARCHAR(100) );'", become=True, check=False, become_user='postgres')
+    assert not s["changed"]
+
+
+def test_postgresql_insert(host):
+    ''' check if we can only write to the primary node of the cluster '''
+
+    s = host.ansible("postgresql_query", "db=test query='INSERT INTO test_ha (name) VALUES (\'test\');'", become=True, check=False, become_user='postgres')
+    assert not s["changed"]

ansible/molecule/ha-pgsql/tests/test_postgres_e_shutdown_primary.py

+import os
+
+import testinfra.utils.ansible_runner
+
+import time
+
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("db0-default")
+
+
+def test_postgresql_check_shutdown_primary(host):
+    ''' Shutdown the primary server '''
+
+    s = host.ansible("command", "systemctl stop postgresql", become=True, check=False)
+    assert s['changed']
+
+    time.sleep(40)
+
+    s = host.socket("tcp://127.0.0.1:5432")
+    assert not s.is_listening

ansible/molecule/ha-pgsql/tests/test_postgres_f_new_primary_cluster_status.py

+import os
+
+import testinfra.utils.ansible_runner
+
+import commons
+
+# /!\ This test run accross all servers
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres")
+
+
+def test_postgresql_check_repmgr_new_master(host):
+    ''' check repmgr status for each node after new master election '''
+
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db0-default"):
+        data = commons.get_status(host)
+        assert data == "fenced"
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db1-default"):
+        data = commons.get_status(host)
+        assert data == "primary"
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db2-default"):
+        data = commons.get_status(host)
+        assert data == "witness"

ansible/molecule/ha-pgsql/tests/test_postgres_g_new_primary_write.py

+import os
+
+import testinfra.utils.ansible_runner
+
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("db1-default")
+
+
+def test_postgresql_insert_new_master(host):
+    s = host.ansible("postgresql_query", "db=test query='INSERT INTO test_ha (name) VALUES (\'test2\');'", become=True, check=False, become_user='postgres')
+    assert s["changed"]

ansible/molecule/ha-pgsql/tests/test_postgres_h_reintegrate_server.py

+import os
+
+import testinfra.utils.ansible_runner
+import time
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("db0-default")
+
+
+def test_postgresql_delete_data(host):
+    ''' delete data directory '''
+
+    s = host.ansible("command", "rm -rf /var/lib/postgresql/11/main/", become=True, check=False)
+    assert s['changed']
+
+
+def test_postgresql_launch_repmgr_sync(host):
+    ''' sync data with primary server using repmgr '''
+
+    current_master = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_host("db1-default")
+    current_master_ip = current_master.interface('eth0').addresses[0]
+    rep_mgr_command = "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby clone -h " + current_master_ip + " -d repmgr -U repmgr -c"
+    s = host.ansible("command", rep_mgr_command, become=True, become_user='postgres', check=False)
+    assert s['changed']
+
+
+def test_postgresql_start_postgresql(host):
+    ''' start postgresql '''
+
+    s = host.ansible("command", "systemctl start postgresql", become=True, check=False)
+    time.sleep(20)
+    assert s['changed']
+
+
+def test_pogresql_register_as_standby(host):
+    ''' register server as standby in repmgr '''
+
+    s = host.ansible("command", "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby register", become=True, become_user='postgres', check=False)
+    time.sleep(20)
+    assert s['changed']

ansible/molecule/ha-pgsql/tests/test_postgres_i_final_cluster_status.py

+import os
+
+import testinfra.utils.ansible_runner
+
+import commons
+
+# /!\ This test run accross all servers
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(os.environ["MOLECULE_INVENTORY_FILE"]).get_hosts("postgres")
+
+
+def test_postgresql_check_status_after_shutdown(host):
+    ''' check repmgr status accross server after primary change and server reintegration '''
+
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db0-default"):
+        data = commons.get_status(host)
+        assert data == "standby"
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db1-default"):
+        data = commons.get_status(host)
+        assert data == "primary"
+    if host.ansible.get_variables()["inventory_hostname"].startswith("db2-default"):
+        data = commons.get_status(host)
+        assert data == "witness"

ansible/playbooks/postgres-maintenance.yml

+#!/usr/bin/env ansible-playbook
+---
+
+- name: GATHER FACTS
+  hosts: postgres_primary:postgres_standby:postgres_fenced
+  tags: always
+  tasks:
+    - name: get cluster state
+      command: "rephacheck"
+      register: rephacheck
+    - name: show status for each node
+      debug:
+        msg: "Current node {{ ansible_hostname }} status {{ rephacheck['stdout'] }}"
+      when: rephacheck['stdout'] != ""
+
+- name: POSTGRESQL SWITCH CURRENT STANDBY TO PRIMARY
+  hosts: postgres_standby
+  tags: [ 'never', 'standby-to-primary' ]
+  tasks:
+    - name: fail if node status if not standby
+      fail:
+        msg: "Current status {{ rephacheck['stdout'] }} must be standby."
+      when: rephacheck['stdout'] != "standby"
+    - name: check if node is currently in standby
+      command: "repmgr standby switchover -f /etc/postgresql/11/main/repmgr.conf --siblings-follow --dry-run"
+      become: yes
+      become_user: postgres
+      when: rephacheck['stdout'] == "standby"
+      register: standby_dry_run
+    - name: switch standby node to primary
+      command: "repmgr standby switchover -f /etc/postgresql/11/main/repmgr.conf --siblings-follow"
+      become: yes
+      become_user: postgres
+      when:
+        - standby_dry_run is succeeded
+        - rephacheck['stdout'] == "standby"
+
+- name: POSTGRESQL SWITCH CURRENT FENCED TO STANDBY
+  hosts: postgres_fenced
+  tags: [ 'never' , 'fenced-to-standby' ]
+  tasks:
+    - name: fail if node status if not fenced
+      fail:
+        msg: "Current status {{ rephacheck['stdout'] }} must be fenced."
+      when: rephacheck['stdout'] != "fenced"
+    - name: stop postgresql
+      systemd:
+        name: postgresql
+        state: stopped
+    - name: delete postgresql data directory
+      file:
+        path: /var/lib/postgresql/11/main/
+        state: absent
+        force: yes
+    - name: copy data from primary
+      command: "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby clone -h {{ hostvars[groups['postgres_primary'][0]]['ansible_default_ipv4']['address'] }} -d repmgr -U repmgr -c"
+      become: yes
+      become_user: postgres
+      register: copy_from_primary
+    - name: start postgresql
+      systemd:
+        name: postgresql
+        state: started
+      when: copy_from_primary is succeeded
+    - name: register node as standby
+      command: "repmgr -f /etc/postgresql/11/main/repmgr.conf --force --verbose standby register"
+      become: yes
+      become_user: postgres
+      when: copy_from_primary is succeeded
+...

ansible/playbooks/site.yml

@@ -8,7 +8,7 @@
     - name: ensure python3 is installed
       register: python_install
       changed_when: "'es_pyinstall' in python_install.stdout_lines"
-      raw: command -v python3 || echo es_pyinstall && apt update && apt install -y python3-minimal python3-apt
+      raw: command -v python3 || echo es_pyinstall && apt update && apt install -y python3-minimal python3-apt iproute2
       tags: always
 - import_playbook: "{{ 'postgres-ha' if groups['postgres']|d('') | length > 1 else 'postgres' }}.yml"
   tags: postgres

ansible/playbooks/site_docker.yml

+---
+- name: DOCKER CONTAINERS PROVISIONING
+  hosts: localhost
+  connection: local
+  tags: always
+  tasks:
+    - name: Create docker containers from inventory
+      docker_container:
+        name: "{{ item }}"
+        image: registry.ubicast.net/docker/debian-systemd:buster
+        privileged: true
+        command: /lib/systemd/systemd
+        state: started
+        volumes:
+          - /sys/fs/cgroup:/sys/fs/cgroup:ro
+        tmpfs:
+          - /tmp
+          - /run
+      with_inventory_hostnames:
+        - all:!localhost
+
+    - name: add host to inventory
+      add_host:
+        name: "{{ item }}"
+        ansible_host: "{{ item }}"
+        ansible_connection: docker
+        ansible_python_interpreter: /usr/bin/python3
+      with_inventory_hostnames:
+        - all:!localhost
+
+- import_playbook: site.yml
+
+...

ansible/requirements.dev.in

 -r requirements.in
 ansible-lint
 flake8
-molecule[docker]
+git+git://github.com/atmaniak/molecule@e03437923b302fca1bd7b4f6030c6956ad00367a#egg=molecule[docker]
+#molecule[docker]
 pip-tools
 testinfra
 yamllint

ansible/requirements.dev.txt

@@ -27,12 +27,11 @@ fasteners==0.15           # via python-gilt
 flake8==3.7.9             # via -r requirements.dev.in
 future==0.18.2            # via cookiecutter
 idna==2.9                 # via requests
-importlib-metadata==1.6.0  # via pluggy, pytest
 jinja2-time==0.2.0        # via cookiecutter
 jinja2==2.11.2            # via ansible, click-completion, cookiecutter, jinja2-time, molecule
 markupsafe==1.1.1         # via jinja2
 mccabe==0.6.1             # via flake8
-molecule[docker]==3.0.3   # via -r requirements.dev.in
+git+git://github.com/atmaniak/molecule@e03437923b302fca1bd7b4f6030c6956ad00367a#egg=molecule[docker]  # via -r requirements.dev.in
 monotonic==1.5            # via fasteners
 more-itertools==8.2.0     # via pytest
 netaddr==0.7.19           # via -r requirements.in
@@ -58,7 +57,7 @@ requests==2.23.0          # via cookiecutter, docker
 ruamel.yaml.clib==0.2.0   # via ruamel.yaml
 ruamel.yaml==0.16.10      # via ansible-lint
 selinux==0.2.1            # via molecule
-sh==1.12.14               # via molecule, python-gilt
+sh==1.13.1                # via molecule, python-gilt
 shellingham==1.3.2        # via click-completion
 six==1.14.0               # via ansible-lint, bcrypt, click-completion, cryptography, docker, fasteners, packaging, pip-tools, pynacl, python-dateutil, websocket-client
 tabulate==0.8.7           # via molecule
@@ -69,7 +68,6 @@ wcwidth==0.1.9            # via pytest
 websocket-client==0.57.0  # via docker
 whichcraft==0.6.1         # via cookiecutter
 yamllint==1.22.1          # via -r requirements.dev.in, molecule
-zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip