From 6b5280a8e45477f480f256d68b1a9d08d97d6db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Diemer?= <stephane.diemer@ubicast.eu> Date: Tue, 29 Sep 2020 13:42:48 +0200 Subject: [PATCH] Reorganize files | refs #32734 --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 4 +- .flake8 | 7 +- .gitignore | 16 +- .gitlab-ci.yml | 2 +- Makefile | 14 +- ansible.cfg => ansible/ansible.cfg | 0 .../example-ha/group_vars/all.yml | 0 .../inventories}/example-ha/host_vars/cs1.yml | 0 .../inventories}/example-ha/host_vars/mi1.yml | 0 .../inventories}/example-ha/host_vars/mm1.yml | 0 .../inventories}/example-ha/host_vars/mo1.yml | 0 .../inventories}/example-ha/host_vars/ms1.yml | 0 .../inventories}/example-ha/host_vars/ms2.yml | 0 .../inventories}/example-ha/host_vars/mv1.yml | 0 .../inventories}/example-ha/host_vars/mw1.yml | 0 .../inventories}/example-ha/host_vars/mw2.yml | 0 .../inventories}/example-ha/host_vars/pg1.yml | 0 .../inventories}/example-ha/host_vars/pg2.yml | 0 .../inventories}/example-ha/host_vars/pg3.yml | 0 .../inventories}/example-ha/host_vars/ws1.yml | 0 .../inventories}/example-ha/hosts | 0 .../inventories}/example/group_vars/all.yml | 0 .../example/host_vars/mymediaserver.yml | 0 .../example/host_vars/mymediavault.yml | 0 .../example/host_vars/mymediaworker.yml | 0 .../example/host_vars/mynetcapture.yml | 0 .../inventories}/example/hosts | 0 .../local-full/host_vars/localhost.dist.yml | 0 .../inventories}/local-full/hosts | 0 .../host_vars/localhost.dist.yml | 0 .../inventories}/local-mediaimport/hosts | 0 .../host_vars/localhost.dist.yml | 0 .../inventories}/local-mediaserver/hosts | 0 .../host_vars/localhost.dist.yml | 0 .../inventories}/local-mediavault/hosts | 0 .../host_vars/localhost.dist.yml | 0 .../inventories}/local-mediaworker/hosts | 0 .../host_vars/localhost.dist.yml | 0 .../inventories}/offline-mediaserver/hosts | 0 .../host_vars/localhost.dist.yml | 0 .../inventories}/offline-mediaworker/hosts | 0 {library => ansible/library}/nmcli.py | 17 +- {library => ansible/library}/source_file.py | 10 +- .../molecule}/default/Dockerfile.j2 | 0 .../molecule}/default/converge.yml | 0 .../molecule}/default/molecule.yml | 0 .../molecule}/default/tests/test_celerity.py | 0 .../molecule}/default/tests/test_conf.py | 0 .../molecule}/default/tests/test_init.py | 0 .../default/tests/test_mediaimport.py | 0 .../default/tests/test_mediaserver.py | 0 .../default/tests/test_mediavault.py | 0 .../default/tests/test_mediaworker.py | 0 .../default/tests/test_mirismanager.py | 0 .../molecule}/default/tests/test_msmonitor.py | 0 .../default/tests/test_netcapture.py | 0 .../molecule}/default/tests/test_nginx.py | 0 .../molecule}/default/tests/test_ntp.py | 0 .../molecule}/default/tests/test_postfix.py | 0 .../molecule}/default/tests/test_postgres.py | 0 .../molecule}/default/tests/test_python3.py | 0 {playbooks => ansible/playbooks}/base.yml | 0 {playbooks => ansible/playbooks}/bench.yml | 0 {playbooks => ansible/playbooks}/celerity.yml | 0 {playbooks => ansible/playbooks}/cluster.yml | 0 .../playbooks}/letsencrypt.yml | 0 .../playbooks}/mediaimport.yml | 0 .../playbooks}/mediaserver.yml | 0 .../playbooks}/mediavault.yml | 0 .../playbooks}/mediaworker.yml | 0 .../playbooks}/migrate-debian.yml | 0 .../playbooks}/mirismanager.yml | 0 .../playbooks}/msmonitor.yml | 0 .../playbooks}/netcapture.yml | 0 {playbooks => ansible/playbooks}/postfix.yml | 0 .../playbooks}/postgres-ha.yml | 0 {playbooks => ansible/playbooks}/postgres.yml | 0 {playbooks => ansible/playbooks}/repos.yml | 0 site.yml => ansible/playbooks/site.yml | 20 +- {playbooks => ansible/playbooks}/tests.yml | 0 {playbooks => ansible/playbooks}/upgrade.yml | 0 {playbooks => ansible/playbooks}/users.yml | 0 {playbooks => ansible/playbooks}/wowza-ha.yml | 0 {playbooks => ansible/playbooks}/wowza.yml | 0 .../plugins}/action/source_file.py | 0 .../requirements.dev.in | 0 .../requirements.dev.txt | 0 requirements.in => ansible/requirements.in | 0 requirements.txt => ansible/requirements.txt | 0 requirements.yml => ansible/requirements.yml | 0 {roles => ansible/roles}/base/meta/main.yml | 0 .../roles}/bench-server/defaults/main.yml | 0 .../roles}/bench-server/meta/main.yml | 0 .../roles}/bench-server/tasks/main.yml | 0 .../templates/bench-streaming.conf.j2 | 0 .../roles}/bench-worker/defaults/main.yml | 0 .../roles}/bench-worker/meta/main.yml | 0 .../roles}/bench-worker/tasks/main.yml | 0 .../roles}/celerity/defaults/main.yml | 0 .../roles}/celerity/handlers/main.yml | 0 .../roles}/celerity/meta/main.yml | 0 .../roles}/celerity/tasks/main.yml | 0 .../celerity/templates/celerity-config.py.j2 | 0 .../roles}/ceph-rbd/defaults/main.yml | 0 .../roles}/ceph-rbd/handlers/main.yml | 0 .../roles}/ceph-rbd/tasks/main.yml | 0 .../templates/ceph.client.user.keyring.j2 | 0 .../roles}/ceph-rbd/templates/ceph.conf.j2 | 0 .../roles}/cluster/defaults/main.yml | 0 .../roles}/cluster/handlers/main.yml | 0 .../roles}/cluster/tasks/main.yml | 0 .../roles}/cluster/templates/corosync.conf.j2 | 0 .../roles}/conf/defaults/main.yml | 0 {roles => ansible/roles}/conf/tasks/main.yml | 0 .../roles}/elastic/defaults/main.yml | 0 .../roles}/elastic/handlers/main.yml | 0 .../roles}/elastic/tasks/main.yml | 0 .../elastic/templates/apm-server.yml.j2 | 0 .../roles}/elastic/templates/kibana.yml.j2 | 0 .../roles}/fail2ban/defaults/main.yml | 0 .../roles}/fail2ban/handlers/main.yml | 0 .../roles}/fail2ban/tasks/main.yml | 0 .../roles}/fail2ban/templates/jail.local.j2 | 0 .../roles}/ferm-configure/defaults/main.yml | 0 .../roles}/ferm-configure/handlers/main.yml | 0 .../roles}/ferm-configure/tasks/main.yml | 0 .../templates/ferm_rules_forward.conf.j2 | 0 .../templates/ferm_rules_input.conf.j2 | 0 .../templates/ferm_rules_output.conf.j2 | 0 .../roles}/ferm-install/defaults/main.yml | 0 .../roles}/ferm-install/handlers/main.yml | 0 .../roles}/ferm-install/tasks/main.yml | 0 .../ferm-install/templates/ferm.conf.j2 | 0 .../roles}/haproxy/defaults/main.yml | 0 .../roles}/haproxy/handlers/main.yml | 0 .../roles}/haproxy/tasks/main.yml | 0 .../roles}/haproxy/templates/haproxy.cfg.j2 | 0 .../roles}/init/defaults/main.yml | 0 {roles => ansible/roles}/init/tasks/main.yml | 0 .../roles}/letsencrypt/defaults/main.yml | 0 .../roles}/letsencrypt/handlers/main.yml | 0 .../roles}/letsencrypt/tasks/main.yml | 0 .../roles}/mediaimport/defaults/main.yml | 0 .../roles}/mediaimport/files/mediaimport | 0 .../roles}/mediaimport/files/mediaimport.py | 0 .../roles}/mediaimport/files/on-upload | Bin .../roles}/mediaimport/files/on-upload.go | 0 .../roles}/mediaimport/handlers/main.yml | 0 .../roles}/mediaimport/meta/main.yml | 0 .../roles}/mediaimport/tasks/main.yml | 0 .../mediaimport/templates/mediaimport.json.j2 | 0 .../mediaimport/templates/sftp_config.j2 | 0 .../roles}/mediaserver/defaults/main.yml | 0 .../roles}/mediaserver/handlers/main.yml | 0 .../roles}/mediaserver/meta/main.yml | 0 .../roles}/mediaserver/tasks/main.yml | 0 .../templates/celerity-config.py.j2 | 0 .../roles}/mediavault/defaults/main.yml | 0 .../roles}/mediavault/handlers/main.yml | 0 .../roles}/mediavault/meta/main.yml | 0 .../roles}/mediavault/tasks/main.yml | 0 .../templates/systemd-backup-service.j2 | 0 .../templates/systemd-backup-timer.j2 | 0 .../templates/systemd-mailer-script.j2 | 0 .../templates/systemd-mailer-service.j2 | 0 .../roles}/mediaworker/defaults/main.yml | 0 .../roles}/mediaworker/handlers/main.yml | 0 .../roles}/mediaworker/meta/main.yml | 0 .../roles}/mediaworker/tasks/main.yml | 0 .../templates/celerity-config.py.j2 | 0 .../roles}/metricbeat/defaults/main.yml | 0 .../roles}/metricbeat/handlers/main.yml | 0 .../roles}/metricbeat/tasks/main.yml | 0 .../metricbeat/templates/metricbeat.yml.j2 | 0 .../metricbeat/templates/postgresql.yml.j2 | 0 .../roles}/mirismanager/defaults/main.yml | 0 .../roles}/mirismanager/files/set_site_url.py | 0 .../roles}/mirismanager/handlers/main.yml | 0 .../roles}/mirismanager/meta/main.yml | 0 .../roles}/mirismanager/tasks/main.yml | 0 .../roles}/msmonitor/defaults/main.yml | 0 .../roles}/msmonitor/handlers/main.yml | 0 .../roles}/msmonitor/meta/main.yml | 0 .../roles}/msmonitor/tasks/main.yml | 0 .../roles}/netcapture/defaults/main.yml | 0 .../roles}/netcapture/meta/main.yml | 0 .../roles}/netcapture/tasks/main.yml | 0 .../netcapture/templates/miris-api.json.j2 | 0 .../netcapture/templates/netcapture.json.j2 | 0 .../roles}/network/defaults/main.yml | 0 .../roles}/network/tasks/main.yml | 0 .../roles}/nginx/defaults/main.yml | 0 .../roles}/nginx/handlers/main.yml | 0 {roles => ansible/roles}/nginx/tasks/main.yml | 0 .../roles}/ocfs2/defaults/main.yml | 0 .../roles}/ocfs2/handlers/main.yml | 0 {roles => ansible/roles}/ocfs2/tasks/main.yml | 0 .../roles}/ocfs2/templates/cluster.conf.j2 | 0 .../roles}/postfix/defaults/main.yml | 0 .../roles}/postfix/handlers/main.yml | 0 .../roles}/postfix/meta/main.yml | 0 .../roles}/postfix/tasks/main.yml | 0 .../roles}/postfix/templates/main.cf.j2 | 0 .../roles}/postgres-ha/defaults/main.yml | 0 .../roles}/postgres-ha/handlers/main.yml | 0 .../roles}/postgres-ha/tasks/main.yml | 0 .../postgres-ha/templates/rephacheck.conf.j2 | 0 .../postgres-ha/templates/rephacheck.py.j2 | 0 .../postgres-ha/templates/repmgr-event.py.j2 | 0 .../postgres-ha/templates/repmgr.conf.j2 | 0 .../roles}/postgres/defaults/main.yml | 0 .../roles}/postgres/handlers/main.yml | 0 .../roles}/postgres/meta/main.yml | 0 .../roles}/postgres/tasks/main.yml | 0 .../roles}/postgres/templates/pg_hba.conf.j2 | 0 .../roles}/proxy/defaults/main.yml | 0 {roles => ansible/roles}/proxy/tasks/main.yml | 0 .../roles}/sysconfig/defaults/main.yml | 0 .../roles}/sysconfig/handlers/main.yml | 0 .../roles}/sysconfig/tasks/locale.yml | 0 .../roles}/sysconfig/tasks/logs.yml | 0 .../roles}/sysconfig/tasks/main.yml | 0 .../roles}/sysconfig/tasks/ntp.yml | 0 .../roles}/sysconfig/tasks/repos.yml | 0 .../roles}/sysconfig/templates/ntp.conf.j2 | 0 .../roles}/users/defaults/main.yml | 0 {roles => ansible/roles}/users/files/.bashrc | 0 {roles => ansible/roles}/users/files/.vimrc | 0 .../roles}/users/files/ubicast_support.pub | 0 .../roles}/users/handlers/main.yml | 0 {roles => ansible/roles}/users/tasks/main.yml | 0 .../roles}/wowza/defaults/main.yml | 0 .../roles}/wowza/handlers/main.yml | 0 {roles => ansible/roles}/wowza/meta/main.yml | 0 {roles => ansible/roles}/wowza/tasks/main.yml | 0 .../roles}/wowza/templates/Server.xml.j2 | 0 .../roles}/wowza/templates/Tune.xml.j2 | 0 .../roles}/wowza/templates/VHost.xml.j2 | 0 .../wowza/templates/live-application.xml.j2 | 0 tester.py | 572 +----------------- pkgs_envsetup.py => tests/pkgs_envsetup.py | 0 __init__.py => tests/scripts/__init__.py | 0 tests/{ => scripts}/test_apt.py | 52 +- tests/{ => scripts}/test_apt_proxy.py | 14 +- tests/{ => scripts}/test_backup.py | 35 +- tests/{ => scripts}/test_dns_records.py | 27 +- tests/{ => scripts}/test_email.py | 55 +- tests/{ => scripts}/test_fail2ban.py | 19 +- tests/{ => scripts}/test_mediaworker.py | 48 +- tests/{ => scripts}/test_monitoring.py | 16 +- tests/{ => scripts}/test_nginx_conf_valid.sh | 0 tests/{ => scripts}/test_nginx_status.py | 8 +- tests/{ => scripts}/test_nginx_vhosts.py | 27 +- tests/{ => scripts}/test_ntp.py | 19 +- tests/{ => scripts}/test_partitions.py | 0 tests/{ => scripts}/test_postgresql.py | 45 +- tests/{ => scripts}/test_raid.py | 0 tests/{ => scripts}/test_ssl.py | 19 +- tests/{ => scripts}/test_wowza.py | 24 +- tests/tester.py | 546 +++++++++++++++++ .../update_envsetup.py | 4 +- logs/.gitkeep => tests/utilities/__init__.py | 0 {utils_lib => tests/utilities}/apt.py | 4 +- {utils_lib => tests/utilities}/commands.py | 0 {utils_lib => tests/utilities}/config.py | 2 +- {utils_lib => tests/utilities}/logging.py | 0 {utils_lib => tests/utilities}/network.py | 0 {utils_lib => tests/utilities}/os.py | 0 getenvsetup.sh => tools/getenvsetup.sh | 0 .../kernels_cleaner.py | 0 {packer => tools/packer}/example.json | 0 {packer => tools/packer}/files/preseed.cfg | 0 {packer => tools/packer}/files/root.cfg | 0 {packer => tools/packer}/files/support.pub | 0 {packer => tools/packer}/scripts/root.sh | 0 {packer => tools/packer}/scripts/upgrade.sh | 0 set_app_domain.py => tools/set_app_domain.py | 46 +- utils.py | 13 - utils_lib/__init__.py | 3 - 280 files changed, 830 insertions(+), 860 deletions(-) rename ansible.cfg => ansible/ansible.cfg (100%) rename {inventories => ansible/inventories}/example-ha/group_vars/all.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/cs1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/mi1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/mm1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/mo1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/ms1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/ms2.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/mv1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/mw1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/mw2.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/pg1.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/pg2.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/pg3.yml (100%) rename {inventories => ansible/inventories}/example-ha/host_vars/ws1.yml (100%) rename {inventories => ansible/inventories}/example-ha/hosts (100%) rename {inventories => ansible/inventories}/example/group_vars/all.yml (100%) rename {inventories => ansible/inventories}/example/host_vars/mymediaserver.yml (100%) rename {inventories => ansible/inventories}/example/host_vars/mymediavault.yml (100%) rename {inventories => ansible/inventories}/example/host_vars/mymediaworker.yml (100%) rename {inventories => ansible/inventories}/example/host_vars/mynetcapture.yml (100%) rename {inventories => ansible/inventories}/example/hosts (100%) rename {inventories => ansible/inventories}/local-full/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/local-full/hosts (100%) rename {inventories => ansible/inventories}/local-mediaimport/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/local-mediaimport/hosts (100%) rename {inventories => ansible/inventories}/local-mediaserver/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/local-mediaserver/hosts (100%) rename {inventories => ansible/inventories}/local-mediavault/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/local-mediavault/hosts (100%) rename {inventories => ansible/inventories}/local-mediaworker/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/local-mediaworker/hosts (100%) rename {inventories => ansible/inventories}/offline-mediaserver/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/offline-mediaserver/hosts (100%) rename {inventories => ansible/inventories}/offline-mediaworker/host_vars/localhost.dist.yml (100%) rename {inventories => ansible/inventories}/offline-mediaworker/hosts (100%) rename {library => ansible/library}/nmcli.py (99%) rename {library => ansible/library}/source_file.py (95%) rename {molecule => ansible/molecule}/default/Dockerfile.j2 (100%) rename {molecule => ansible/molecule}/default/converge.yml (100%) rename {molecule => ansible/molecule}/default/molecule.yml (100%) rename {molecule => ansible/molecule}/default/tests/test_celerity.py (100%) rename {molecule => ansible/molecule}/default/tests/test_conf.py (100%) rename {molecule => ansible/molecule}/default/tests/test_init.py (100%) rename {molecule => ansible/molecule}/default/tests/test_mediaimport.py (100%) rename {molecule => ansible/molecule}/default/tests/test_mediaserver.py (100%) rename {molecule => ansible/molecule}/default/tests/test_mediavault.py (100%) rename {molecule => ansible/molecule}/default/tests/test_mediaworker.py (100%) rename {molecule => ansible/molecule}/default/tests/test_mirismanager.py (100%) rename {molecule => ansible/molecule}/default/tests/test_msmonitor.py (100%) rename {molecule => ansible/molecule}/default/tests/test_netcapture.py (100%) rename {molecule => ansible/molecule}/default/tests/test_nginx.py (100%) rename {molecule => ansible/molecule}/default/tests/test_ntp.py (100%) rename {molecule => ansible/molecule}/default/tests/test_postfix.py (100%) rename {molecule => ansible/molecule}/default/tests/test_postgres.py (100%) rename {molecule => ansible/molecule}/default/tests/test_python3.py (100%) rename {playbooks => ansible/playbooks}/base.yml (100%) rename {playbooks => ansible/playbooks}/bench.yml (100%) rename {playbooks => ansible/playbooks}/celerity.yml (100%) rename {playbooks => ansible/playbooks}/cluster.yml (100%) rename {playbooks => ansible/playbooks}/letsencrypt.yml (100%) rename {playbooks => ansible/playbooks}/mediaimport.yml (100%) rename {playbooks => ansible/playbooks}/mediaserver.yml (100%) rename {playbooks => ansible/playbooks}/mediavault.yml (100%) rename {playbooks => ansible/playbooks}/mediaworker.yml (100%) rename {playbooks => ansible/playbooks}/migrate-debian.yml (100%) rename {playbooks => ansible/playbooks}/mirismanager.yml (100%) rename {playbooks => ansible/playbooks}/msmonitor.yml (100%) rename {playbooks => ansible/playbooks}/netcapture.yml (100%) rename {playbooks => ansible/playbooks}/postfix.yml (100%) rename {playbooks => ansible/playbooks}/postgres-ha.yml (100%) rename {playbooks => ansible/playbooks}/postgres.yml (100%) rename {playbooks => ansible/playbooks}/repos.yml (100%) rename site.yml => ansible/playbooks/site.yml (51%) rename {playbooks => ansible/playbooks}/tests.yml (100%) rename {playbooks => ansible/playbooks}/upgrade.yml (100%) rename {playbooks => ansible/playbooks}/users.yml (100%) rename {playbooks => ansible/playbooks}/wowza-ha.yml (100%) rename {playbooks => ansible/playbooks}/wowza.yml (100%) rename {plugins => ansible/plugins}/action/source_file.py (100%) rename requirements.dev.in => ansible/requirements.dev.in (100%) rename requirements.dev.txt => ansible/requirements.dev.txt (100%) rename requirements.in => ansible/requirements.in (100%) rename requirements.txt => ansible/requirements.txt (100%) rename requirements.yml => ansible/requirements.yml (100%) rename {roles => ansible/roles}/base/meta/main.yml (100%) rename {roles => ansible/roles}/bench-server/defaults/main.yml (100%) rename {roles => ansible/roles}/bench-server/meta/main.yml (100%) rename {roles => ansible/roles}/bench-server/tasks/main.yml (100%) rename {roles => ansible/roles}/bench-server/templates/bench-streaming.conf.j2 (100%) rename {roles => ansible/roles}/bench-worker/defaults/main.yml (100%) rename {roles => ansible/roles}/bench-worker/meta/main.yml (100%) rename {roles => ansible/roles}/bench-worker/tasks/main.yml (100%) rename {roles => ansible/roles}/celerity/defaults/main.yml (100%) rename {roles => ansible/roles}/celerity/handlers/main.yml (100%) rename {roles => ansible/roles}/celerity/meta/main.yml (100%) rename {roles => ansible/roles}/celerity/tasks/main.yml (100%) rename {roles => ansible/roles}/celerity/templates/celerity-config.py.j2 (100%) rename {roles => ansible/roles}/ceph-rbd/defaults/main.yml (100%) rename {roles => ansible/roles}/ceph-rbd/handlers/main.yml (100%) rename {roles => ansible/roles}/ceph-rbd/tasks/main.yml (100%) rename {roles => ansible/roles}/ceph-rbd/templates/ceph.client.user.keyring.j2 (100%) rename {roles => ansible/roles}/ceph-rbd/templates/ceph.conf.j2 (100%) rename {roles => ansible/roles}/cluster/defaults/main.yml (100%) rename {roles => ansible/roles}/cluster/handlers/main.yml (100%) rename {roles => ansible/roles}/cluster/tasks/main.yml (100%) rename {roles => ansible/roles}/cluster/templates/corosync.conf.j2 (100%) rename {roles => ansible/roles}/conf/defaults/main.yml (100%) rename {roles => ansible/roles}/conf/tasks/main.yml (100%) rename {roles => ansible/roles}/elastic/defaults/main.yml (100%) rename {roles => ansible/roles}/elastic/handlers/main.yml (100%) rename {roles => ansible/roles}/elastic/tasks/main.yml (100%) rename {roles => ansible/roles}/elastic/templates/apm-server.yml.j2 (100%) rename {roles => ansible/roles}/elastic/templates/kibana.yml.j2 (100%) rename {roles => ansible/roles}/fail2ban/defaults/main.yml (100%) rename {roles => ansible/roles}/fail2ban/handlers/main.yml (100%) rename {roles => ansible/roles}/fail2ban/tasks/main.yml (100%) rename {roles => ansible/roles}/fail2ban/templates/jail.local.j2 (100%) rename {roles => ansible/roles}/ferm-configure/defaults/main.yml (100%) rename {roles => ansible/roles}/ferm-configure/handlers/main.yml (100%) rename {roles => ansible/roles}/ferm-configure/tasks/main.yml (100%) rename {roles => ansible/roles}/ferm-configure/templates/ferm_rules_forward.conf.j2 (100%) rename {roles => ansible/roles}/ferm-configure/templates/ferm_rules_input.conf.j2 (100%) rename {roles => ansible/roles}/ferm-configure/templates/ferm_rules_output.conf.j2 (100%) rename {roles => ansible/roles}/ferm-install/defaults/main.yml (100%) rename {roles => ansible/roles}/ferm-install/handlers/main.yml (100%) rename {roles => ansible/roles}/ferm-install/tasks/main.yml (100%) rename {roles => ansible/roles}/ferm-install/templates/ferm.conf.j2 (100%) rename {roles => ansible/roles}/haproxy/defaults/main.yml (100%) rename {roles => ansible/roles}/haproxy/handlers/main.yml (100%) rename {roles => ansible/roles}/haproxy/tasks/main.yml (100%) rename {roles => ansible/roles}/haproxy/templates/haproxy.cfg.j2 (100%) rename {roles => ansible/roles}/init/defaults/main.yml (100%) rename {roles => ansible/roles}/init/tasks/main.yml (100%) rename {roles => ansible/roles}/letsencrypt/defaults/main.yml (100%) rename {roles => ansible/roles}/letsencrypt/handlers/main.yml (100%) rename {roles => ansible/roles}/letsencrypt/tasks/main.yml (100%) rename {roles => ansible/roles}/mediaimport/defaults/main.yml (100%) rename {roles => ansible/roles}/mediaimport/files/mediaimport (100%) rename {roles => ansible/roles}/mediaimport/files/mediaimport.py (100%) rename {roles => ansible/roles}/mediaimport/files/on-upload (100%) rename {roles => ansible/roles}/mediaimport/files/on-upload.go (100%) rename {roles => ansible/roles}/mediaimport/handlers/main.yml (100%) rename {roles => ansible/roles}/mediaimport/meta/main.yml (100%) rename {roles => ansible/roles}/mediaimport/tasks/main.yml (100%) rename {roles => ansible/roles}/mediaimport/templates/mediaimport.json.j2 (100%) rename {roles => ansible/roles}/mediaimport/templates/sftp_config.j2 (100%) rename {roles => ansible/roles}/mediaserver/defaults/main.yml (100%) rename {roles => ansible/roles}/mediaserver/handlers/main.yml (100%) rename {roles => ansible/roles}/mediaserver/meta/main.yml (100%) rename {roles => ansible/roles}/mediaserver/tasks/main.yml (100%) rename {roles => ansible/roles}/mediaserver/templates/celerity-config.py.j2 (100%) rename {roles => ansible/roles}/mediavault/defaults/main.yml (100%) rename {roles => ansible/roles}/mediavault/handlers/main.yml (100%) rename {roles => ansible/roles}/mediavault/meta/main.yml (100%) rename {roles => ansible/roles}/mediavault/tasks/main.yml (100%) rename {roles => ansible/roles}/mediavault/templates/systemd-backup-service.j2 (100%) rename {roles => ansible/roles}/mediavault/templates/systemd-backup-timer.j2 (100%) rename {roles => ansible/roles}/mediavault/templates/systemd-mailer-script.j2 (100%) rename {roles => ansible/roles}/mediavault/templates/systemd-mailer-service.j2 (100%) rename {roles => ansible/roles}/mediaworker/defaults/main.yml (100%) rename {roles => ansible/roles}/mediaworker/handlers/main.yml (100%) rename {roles => ansible/roles}/mediaworker/meta/main.yml (100%) rename {roles => ansible/roles}/mediaworker/tasks/main.yml (100%) rename {roles => ansible/roles}/mediaworker/templates/celerity-config.py.j2 (100%) rename {roles => ansible/roles}/metricbeat/defaults/main.yml (100%) rename {roles => ansible/roles}/metricbeat/handlers/main.yml (100%) rename {roles => ansible/roles}/metricbeat/tasks/main.yml (100%) rename {roles => ansible/roles}/metricbeat/templates/metricbeat.yml.j2 (100%) rename {roles => ansible/roles}/metricbeat/templates/postgresql.yml.j2 (100%) rename {roles => ansible/roles}/mirismanager/defaults/main.yml (100%) rename {roles => ansible/roles}/mirismanager/files/set_site_url.py (100%) rename {roles => ansible/roles}/mirismanager/handlers/main.yml (100%) rename {roles => ansible/roles}/mirismanager/meta/main.yml (100%) rename {roles => ansible/roles}/mirismanager/tasks/main.yml (100%) rename {roles => ansible/roles}/msmonitor/defaults/main.yml (100%) rename {roles => ansible/roles}/msmonitor/handlers/main.yml (100%) rename {roles => ansible/roles}/msmonitor/meta/main.yml (100%) rename {roles => ansible/roles}/msmonitor/tasks/main.yml (100%) rename {roles => ansible/roles}/netcapture/defaults/main.yml (100%) rename {roles => ansible/roles}/netcapture/meta/main.yml (100%) rename {roles => ansible/roles}/netcapture/tasks/main.yml (100%) rename {roles => ansible/roles}/netcapture/templates/miris-api.json.j2 (100%) rename {roles => ansible/roles}/netcapture/templates/netcapture.json.j2 (100%) rename {roles => ansible/roles}/network/defaults/main.yml (100%) rename {roles => ansible/roles}/network/tasks/main.yml (100%) rename {roles => ansible/roles}/nginx/defaults/main.yml (100%) rename {roles => ansible/roles}/nginx/handlers/main.yml (100%) rename {roles => ansible/roles}/nginx/tasks/main.yml (100%) rename {roles => ansible/roles}/ocfs2/defaults/main.yml (100%) rename {roles => ansible/roles}/ocfs2/handlers/main.yml (100%) rename {roles => ansible/roles}/ocfs2/tasks/main.yml (100%) rename {roles => ansible/roles}/ocfs2/templates/cluster.conf.j2 (100%) rename {roles => ansible/roles}/postfix/defaults/main.yml (100%) rename {roles => ansible/roles}/postfix/handlers/main.yml (100%) rename {roles => ansible/roles}/postfix/meta/main.yml (100%) rename {roles => ansible/roles}/postfix/tasks/main.yml (100%) rename {roles => ansible/roles}/postfix/templates/main.cf.j2 (100%) rename {roles => ansible/roles}/postgres-ha/defaults/main.yml (100%) rename {roles => ansible/roles}/postgres-ha/handlers/main.yml (100%) rename {roles => ansible/roles}/postgres-ha/tasks/main.yml (100%) rename {roles => ansible/roles}/postgres-ha/templates/rephacheck.conf.j2 (100%) rename {roles => ansible/roles}/postgres-ha/templates/rephacheck.py.j2 (100%) rename {roles => ansible/roles}/postgres-ha/templates/repmgr-event.py.j2 (100%) rename {roles => ansible/roles}/postgres-ha/templates/repmgr.conf.j2 (100%) rename {roles => ansible/roles}/postgres/defaults/main.yml (100%) rename {roles => ansible/roles}/postgres/handlers/main.yml (100%) rename {roles => ansible/roles}/postgres/meta/main.yml (100%) rename {roles => ansible/roles}/postgres/tasks/main.yml (100%) rename {roles => ansible/roles}/postgres/templates/pg_hba.conf.j2 (100%) rename {roles => ansible/roles}/proxy/defaults/main.yml (100%) rename {roles => ansible/roles}/proxy/tasks/main.yml (100%) rename {roles => ansible/roles}/sysconfig/defaults/main.yml (100%) rename {roles => ansible/roles}/sysconfig/handlers/main.yml (100%) rename {roles => ansible/roles}/sysconfig/tasks/locale.yml (100%) rename {roles => ansible/roles}/sysconfig/tasks/logs.yml (100%) rename {roles => ansible/roles}/sysconfig/tasks/main.yml (100%) rename {roles => ansible/roles}/sysconfig/tasks/ntp.yml (100%) rename {roles => ansible/roles}/sysconfig/tasks/repos.yml (100%) rename {roles => ansible/roles}/sysconfig/templates/ntp.conf.j2 (100%) rename {roles => ansible/roles}/users/defaults/main.yml (100%) rename {roles => ansible/roles}/users/files/.bashrc (100%) rename {roles => ansible/roles}/users/files/.vimrc (100%) rename {roles => ansible/roles}/users/files/ubicast_support.pub (100%) rename {roles => ansible/roles}/users/handlers/main.yml (100%) rename {roles => ansible/roles}/users/tasks/main.yml (100%) rename {roles => ansible/roles}/wowza/defaults/main.yml (100%) rename {roles => ansible/roles}/wowza/handlers/main.yml (100%) rename {roles => ansible/roles}/wowza/meta/main.yml (100%) rename {roles => ansible/roles}/wowza/tasks/main.yml (100%) rename {roles => ansible/roles}/wowza/templates/Server.xml.j2 (100%) rename {roles => ansible/roles}/wowza/templates/Tune.xml.j2 (100%) rename {roles => ansible/roles}/wowza/templates/VHost.xml.j2 (100%) rename {roles => ansible/roles}/wowza/templates/live-application.xml.j2 (100%) mode change 100755 => 120000 tester.py rename pkgs_envsetup.py => tests/pkgs_envsetup.py (100%) rename __init__.py => tests/scripts/__init__.py (100%) rename tests/{ => scripts}/test_apt.py (73%) rename tests/{ => scripts}/test_apt_proxy.py (77%) rename tests/{ => scripts}/test_backup.py (86%) rename tests/{ => scripts}/test_dns_records.py (86%) rename tests/{ => scripts}/test_email.py (80%) rename tests/{ => scripts}/test_fail2ban.py (88%) rename tests/{ => scripts}/test_mediaworker.py (72%) rename tests/{ => scripts}/test_monitoring.py (74%) rename tests/{ => scripts}/test_nginx_conf_valid.sh (100%) rename tests/{ => scripts}/test_nginx_status.py (81%) rename tests/{ => scripts}/test_nginx_vhosts.py (87%) rename tests/{ => scripts}/test_ntp.py (73%) rename tests/{ => scripts}/test_partitions.py (100%) rename tests/{ => scripts}/test_postgresql.py (88%) rename tests/{ => scripts}/test_raid.py (100%) rename tests/{ => scripts}/test_ssl.py (82%) rename tests/{ => scripts}/test_wowza.py (84%) create mode 100755 tests/tester.py rename update_envsetup.py => tests/update_envsetup.py (90%) rename logs/.gitkeep => tests/utilities/__init__.py (100%) rename {utils_lib => tests/utilities}/apt.py (98%) rename {utils_lib => tests/utilities}/commands.py (100%) rename {utils_lib => tests/utilities}/config.py (98%) rename {utils_lib => tests/utilities}/logging.py (100%) rename {utils_lib => tests/utilities}/network.py (100%) rename {utils_lib => tests/utilities}/os.py (100%) rename getenvsetup.sh => tools/getenvsetup.sh (100%) rename kernels_cleaner.py => tools/kernels_cleaner.py (100%) rename {packer => tools/packer}/example.json (100%) rename {packer => tools/packer}/files/preseed.cfg (100%) rename {packer => tools/packer}/files/root.cfg (100%) rename {packer => tools/packer}/files/support.pub (100%) rename {packer => tools/packer}/scripts/root.sh (100%) rename {packer => tools/packer}/scripts/upgrade.sh (100%) rename set_app_domain.py => tools/set_app_domain.py (86%) delete mode 100644 utils.py delete mode 100644 utils_lib/__init__.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c526e078..5ed1d892 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -63,7 +63,7 @@ RUN \ && \ : -COPY requirements.dev.txt . +COPY ansible/requirements.dev.txt . RUN \ # ansible & co pip install -r requirements.dev.txt && \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 40321bc5..3c161363 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,8 +16,8 @@ ], "settings": { "files.associations": { - "**/requirements.in/*.txt": "pip-requirements", - "requirements.*.txt": "pip-requirements", + "**/ansible/requirements.in/*.txt": "pip-requirements", + "ansible/requirements.*.txt": "pip-requirements", "**/defaults/**/*": "ansible", "**/tasks/**/*.yml": "ansible", "**/handler/*.yml": "ansible", diff --git a/.flake8 b/.flake8 index 7b5099d7..f9f50e5e 100644 --- a/.flake8 +++ b/.flake8 @@ -6,8 +6,5 @@ ignore = W503 W505 -per-file-ignores = - roles/elastic.elasticsearch/*:E713 - roles/manager/files/set_site_url.py:E402 - library/*:E402 - library/nmcli.py:E402,F401 +#per-file-ignores = +# ansible/roles/elastic.elasticsearch/*:E713 diff --git a/.gitignore b/.gitignore index 6245fd6b..97c5e1e8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,12 +6,14 @@ __pycache__/ *.pyc # ansible -inventories/_* -inventories/local*/host_vars/localhost.yml -inventories/offline*/host_vars/localhost.yml -playbooks/_* -roles/_* -roles/elastic.elasticsearch +ansible/inventories/_* +ansible/inventories/local*/host_vars/localhost.yml +ansible/inventories/offline*/host_vars/localhost.yml +ansible/playbooks/_* +ansible/roles/_* +ansible/roles/elastic.elasticsearch + +# logs ./logs/ log/ @@ -39,5 +41,5 @@ packer/*.json conf*.sh auto-generated-conf.sh* log*.txt -tests/ms-testing-suite +tests/scripts/ms-testing-suite *.log diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76930eb5..4ce71875 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ docker:build: - if: '$CI_PIPELINE_SOURCE == "push"' changes: - .devcontainer/Dockerfile - - requirements.dev.txt + - ansible/requirements.dev.txt before_script: - apk add bash make - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.ubicast.net diff --git a/Makefile b/Makefile index b2c6d1a7..d020a117 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,8 @@ venv: -@command -v apt-get >/dev/null && apt-get update && apt-get install -y python3-venv @command -v $(PIP_BIN) > /dev/null || python3 -m venv $(VENV) -## requirements.txt: Update requirements and their dependencies -## requirements.dev.txt: Update development requirements and their dependencies +## ansible/requirements.txt: Update requirements and their dependencies +## ansible/requirements.dev.txt: Update development requirements and their dependencies %.txt: %.in $(PIP_COMPILE_BIN) -U $^ -o $@ chmod 644 $@ @@ -38,14 +38,14 @@ venv: ## install: Install requirements install: venv $(PIP_BIN) install -U pip wheel - $(PIP_BIN) install -r requirements.txt - $(ANSIBLE_GALAXY_BIN) install -r requirements.yml + $(PIP_BIN) install -r ansible/requirements.txt + $(ANSIBLE_GALAXY_BIN) install -r ansible/requirements.yml .PHONY: install-dev ## install-dev: Install development requirements install-dev: install - $(PIP_BIN) install -r requirements.dev.txt + $(PIP_BIN) install -r ansible/requirements.dev.txt [ -d .git/hooks ] || mkdir .git/hooks ln -sfv .githooks/pre-commit .git/hooks/ || echo "Failed to create pre-commit link" @@ -54,7 +54,7 @@ install-dev: install lint: $(FLAKE8_BIN) . $(YAMLLINT_BIN) . - $(ANSIBLE_LINT_BIN) site.yml + $(ANSIBLE_LINT_BIN) ansible/playbooks/site.yml .PHONY: test ## test: Run development tests on the project : debug=1, keep=1, SKYREACH_SYSTEM_KEY=<xxx> @@ -77,7 +77,7 @@ ifndef t $(eval t=all) endif $(ANSIBLE_BIN) -i $(i) -l $(l) -m ping all - $(ANSIBLE_PLAYBOOK_BIN) -i $(i) site.yml -e conf_update=true -l $(l) -t $(t) + $(ANSIBLE_PLAYBOOK_BIN) -i $(i) ansible/playbooks/site.yml -e conf_update=true -l $(l) -t $(t) .PHONY: image-validate ## image-validate: Check that Packer image is valid : build=<path-to-packer-file> diff --git a/ansible.cfg b/ansible/ansible.cfg similarity index 100% rename from ansible.cfg rename to ansible/ansible.cfg diff --git a/inventories/example-ha/group_vars/all.yml b/ansible/inventories/example-ha/group_vars/all.yml similarity index 100% rename from inventories/example-ha/group_vars/all.yml rename to ansible/inventories/example-ha/group_vars/all.yml diff --git a/inventories/example-ha/host_vars/cs1.yml b/ansible/inventories/example-ha/host_vars/cs1.yml similarity index 100% rename from inventories/example-ha/host_vars/cs1.yml rename to ansible/inventories/example-ha/host_vars/cs1.yml diff --git a/inventories/example-ha/host_vars/mi1.yml b/ansible/inventories/example-ha/host_vars/mi1.yml similarity index 100% rename from inventories/example-ha/host_vars/mi1.yml rename to ansible/inventories/example-ha/host_vars/mi1.yml diff --git a/inventories/example-ha/host_vars/mm1.yml b/ansible/inventories/example-ha/host_vars/mm1.yml similarity index 100% rename from inventories/example-ha/host_vars/mm1.yml rename to ansible/inventories/example-ha/host_vars/mm1.yml diff --git a/inventories/example-ha/host_vars/mo1.yml b/ansible/inventories/example-ha/host_vars/mo1.yml similarity index 100% rename from inventories/example-ha/host_vars/mo1.yml rename to ansible/inventories/example-ha/host_vars/mo1.yml diff --git a/inventories/example-ha/host_vars/ms1.yml b/ansible/inventories/example-ha/host_vars/ms1.yml similarity index 100% rename from inventories/example-ha/host_vars/ms1.yml rename to ansible/inventories/example-ha/host_vars/ms1.yml diff --git a/inventories/example-ha/host_vars/ms2.yml b/ansible/inventories/example-ha/host_vars/ms2.yml similarity index 100% rename from inventories/example-ha/host_vars/ms2.yml rename to ansible/inventories/example-ha/host_vars/ms2.yml diff --git a/inventories/example-ha/host_vars/mv1.yml b/ansible/inventories/example-ha/host_vars/mv1.yml similarity index 100% rename from inventories/example-ha/host_vars/mv1.yml rename to ansible/inventories/example-ha/host_vars/mv1.yml diff --git a/inventories/example-ha/host_vars/mw1.yml b/ansible/inventories/example-ha/host_vars/mw1.yml similarity index 100% rename from inventories/example-ha/host_vars/mw1.yml rename to ansible/inventories/example-ha/host_vars/mw1.yml diff --git a/inventories/example-ha/host_vars/mw2.yml b/ansible/inventories/example-ha/host_vars/mw2.yml similarity index 100% rename from inventories/example-ha/host_vars/mw2.yml rename to ansible/inventories/example-ha/host_vars/mw2.yml diff --git a/inventories/example-ha/host_vars/pg1.yml b/ansible/inventories/example-ha/host_vars/pg1.yml similarity index 100% rename from inventories/example-ha/host_vars/pg1.yml rename to ansible/inventories/example-ha/host_vars/pg1.yml diff --git a/inventories/example-ha/host_vars/pg2.yml b/ansible/inventories/example-ha/host_vars/pg2.yml similarity index 100% rename from inventories/example-ha/host_vars/pg2.yml rename to ansible/inventories/example-ha/host_vars/pg2.yml diff --git a/inventories/example-ha/host_vars/pg3.yml b/ansible/inventories/example-ha/host_vars/pg3.yml similarity index 100% rename from inventories/example-ha/host_vars/pg3.yml rename to ansible/inventories/example-ha/host_vars/pg3.yml diff --git a/inventories/example-ha/host_vars/ws1.yml b/ansible/inventories/example-ha/host_vars/ws1.yml similarity index 100% rename from inventories/example-ha/host_vars/ws1.yml rename to ansible/inventories/example-ha/host_vars/ws1.yml diff --git a/inventories/example-ha/hosts b/ansible/inventories/example-ha/hosts similarity index 100% rename from inventories/example-ha/hosts rename to ansible/inventories/example-ha/hosts diff --git a/inventories/example/group_vars/all.yml b/ansible/inventories/example/group_vars/all.yml similarity index 100% rename from inventories/example/group_vars/all.yml rename to ansible/inventories/example/group_vars/all.yml diff --git a/inventories/example/host_vars/mymediaserver.yml b/ansible/inventories/example/host_vars/mymediaserver.yml similarity index 100% rename from inventories/example/host_vars/mymediaserver.yml rename to ansible/inventories/example/host_vars/mymediaserver.yml diff --git a/inventories/example/host_vars/mymediavault.yml b/ansible/inventories/example/host_vars/mymediavault.yml similarity index 100% rename from inventories/example/host_vars/mymediavault.yml rename to ansible/inventories/example/host_vars/mymediavault.yml diff --git a/inventories/example/host_vars/mymediaworker.yml b/ansible/inventories/example/host_vars/mymediaworker.yml similarity index 100% rename from inventories/example/host_vars/mymediaworker.yml rename to ansible/inventories/example/host_vars/mymediaworker.yml diff --git a/inventories/example/host_vars/mynetcapture.yml b/ansible/inventories/example/host_vars/mynetcapture.yml similarity index 100% rename from inventories/example/host_vars/mynetcapture.yml rename to ansible/inventories/example/host_vars/mynetcapture.yml diff --git a/inventories/example/hosts b/ansible/inventories/example/hosts similarity index 100% rename from inventories/example/hosts rename to ansible/inventories/example/hosts diff --git a/inventories/local-full/host_vars/localhost.dist.yml b/ansible/inventories/local-full/host_vars/localhost.dist.yml similarity index 100% rename from inventories/local-full/host_vars/localhost.dist.yml rename to ansible/inventories/local-full/host_vars/localhost.dist.yml diff --git a/inventories/local-full/hosts b/ansible/inventories/local-full/hosts similarity index 100% rename from inventories/local-full/hosts rename to ansible/inventories/local-full/hosts diff --git a/inventories/local-mediaimport/host_vars/localhost.dist.yml b/ansible/inventories/local-mediaimport/host_vars/localhost.dist.yml similarity index 100% rename from inventories/local-mediaimport/host_vars/localhost.dist.yml rename to ansible/inventories/local-mediaimport/host_vars/localhost.dist.yml diff --git a/inventories/local-mediaimport/hosts b/ansible/inventories/local-mediaimport/hosts similarity index 100% rename from inventories/local-mediaimport/hosts rename to ansible/inventories/local-mediaimport/hosts diff --git a/inventories/local-mediaserver/host_vars/localhost.dist.yml b/ansible/inventories/local-mediaserver/host_vars/localhost.dist.yml similarity index 100% rename from inventories/local-mediaserver/host_vars/localhost.dist.yml rename to ansible/inventories/local-mediaserver/host_vars/localhost.dist.yml diff --git a/inventories/local-mediaserver/hosts b/ansible/inventories/local-mediaserver/hosts similarity index 100% rename from inventories/local-mediaserver/hosts rename to ansible/inventories/local-mediaserver/hosts diff --git a/inventories/local-mediavault/host_vars/localhost.dist.yml b/ansible/inventories/local-mediavault/host_vars/localhost.dist.yml similarity index 100% rename from inventories/local-mediavault/host_vars/localhost.dist.yml rename to ansible/inventories/local-mediavault/host_vars/localhost.dist.yml diff --git a/inventories/local-mediavault/hosts b/ansible/inventories/local-mediavault/hosts similarity index 100% rename from inventories/local-mediavault/hosts rename to ansible/inventories/local-mediavault/hosts diff --git a/inventories/local-mediaworker/host_vars/localhost.dist.yml b/ansible/inventories/local-mediaworker/host_vars/localhost.dist.yml similarity index 100% rename from inventories/local-mediaworker/host_vars/localhost.dist.yml rename to ansible/inventories/local-mediaworker/host_vars/localhost.dist.yml diff --git a/inventories/local-mediaworker/hosts b/ansible/inventories/local-mediaworker/hosts similarity index 100% rename from inventories/local-mediaworker/hosts rename to ansible/inventories/local-mediaworker/hosts diff --git a/inventories/offline-mediaserver/host_vars/localhost.dist.yml b/ansible/inventories/offline-mediaserver/host_vars/localhost.dist.yml similarity index 100% rename from inventories/offline-mediaserver/host_vars/localhost.dist.yml rename to ansible/inventories/offline-mediaserver/host_vars/localhost.dist.yml diff --git a/inventories/offline-mediaserver/hosts b/ansible/inventories/offline-mediaserver/hosts similarity index 100% rename from inventories/offline-mediaserver/hosts rename to ansible/inventories/offline-mediaserver/hosts diff --git a/inventories/offline-mediaworker/host_vars/localhost.dist.yml b/ansible/inventories/offline-mediaworker/host_vars/localhost.dist.yml similarity index 100% rename from inventories/offline-mediaworker/host_vars/localhost.dist.yml rename to ansible/inventories/offline-mediaworker/host_vars/localhost.dist.yml diff --git a/inventories/offline-mediaworker/hosts b/ansible/inventories/offline-mediaworker/hosts similarity index 100% rename from inventories/offline-mediaworker/hosts rename to ansible/inventories/offline-mediaworker/hosts diff --git a/library/nmcli.py b/ansible/library/nmcli.py similarity index 99% rename from library/nmcli.py rename to ansible/library/nmcli.py index 2d6d46ee..01f285ac 100644 --- a/library/nmcli.py +++ b/ansible/library/nmcli.py @@ -6,6 +6,9 @@ # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import absolute_import, division, print_function + +import traceback + __metaclass__ = type ANSIBLE_METADATA = { @@ -14,7 +17,7 @@ ANSIBLE_METADATA = { 'supported_by': 'community' } -DOCUMENTATION = r''' +DOCUMENTATION = ''' --- module: nmcli author: @@ -260,7 +263,7 @@ options: version_added: "2.8" ''' -EXAMPLES = r''' +EXAMPLES = ''' # These examples are using the following inventory: # # ## Directory layout: @@ -559,8 +562,6 @@ EXAMPLES = r''' RETURN = r"""# """ -import traceback - DBUS_IMP_ERR = None try: import dbus @@ -574,19 +575,19 @@ HAVE_NM_CLIENT = True try: import gi gi.require_version('NM', '1.0') - from gi.repository import NM + # from gi.repository import NM except (ImportError, ValueError): try: import gi gi.require_version('NMClient', '1.0') gi.require_version('NetworkManager', '1.0') - from gi.repository import NetworkManager, NMClient + # from gi.repository import NetworkManager, NMClient except (ImportError, ValueError): NM_CLIENT_IMP_ERR = traceback.format_exc() HAVE_NM_CLIENT = False -from ansible.module_utils.basic import AnsibleModule, missing_required_lib -from ansible.module_utils._text import to_native +from ansible.module_utils.basic import AnsibleModule, missing_required_lib # noqa: E402 +from ansible.module_utils._text import to_native # noqa: E402 class Nmcli(object): diff --git a/library/source_file.py b/ansible/library/source_file.py similarity index 95% rename from library/source_file.py rename to ansible/library/source_file.py index 02df176f..54e26007 100644 --- a/library/source_file.py +++ b/ansible/library/source_file.py @@ -61,12 +61,12 @@ ansible_facts: key: value """ -import os -import re +import os # noqa: E402 +import re # noqa: E402 -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.parsing.convert_bool import BOOLEANS, boolean -from ansible.module_utils.six import string_types +from ansible.module_utils.basic import AnsibleModule # noqa: E402 +from ansible.module_utils.parsing.convert_bool import BOOLEANS, boolean # noqa: E402 +from ansible.module_utils.six import string_types # noqa: E402 def run_module(): diff --git a/molecule/default/Dockerfile.j2 b/ansible/molecule/default/Dockerfile.j2 similarity index 100% rename from molecule/default/Dockerfile.j2 rename to ansible/molecule/default/Dockerfile.j2 diff --git a/molecule/default/converge.yml b/ansible/molecule/default/converge.yml similarity index 100% rename from molecule/default/converge.yml rename to ansible/molecule/default/converge.yml diff --git a/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml similarity index 100% rename from molecule/default/molecule.yml rename to ansible/molecule/default/molecule.yml diff --git a/molecule/default/tests/test_celerity.py b/ansible/molecule/default/tests/test_celerity.py similarity index 100% rename from molecule/default/tests/test_celerity.py rename to ansible/molecule/default/tests/test_celerity.py diff --git a/molecule/default/tests/test_conf.py b/ansible/molecule/default/tests/test_conf.py similarity index 100% rename from molecule/default/tests/test_conf.py rename to ansible/molecule/default/tests/test_conf.py diff --git a/molecule/default/tests/test_init.py b/ansible/molecule/default/tests/test_init.py similarity index 100% rename from molecule/default/tests/test_init.py rename to ansible/molecule/default/tests/test_init.py diff --git a/molecule/default/tests/test_mediaimport.py b/ansible/molecule/default/tests/test_mediaimport.py similarity index 100% rename from molecule/default/tests/test_mediaimport.py rename to ansible/molecule/default/tests/test_mediaimport.py diff --git a/molecule/default/tests/test_mediaserver.py b/ansible/molecule/default/tests/test_mediaserver.py similarity index 100% rename from molecule/default/tests/test_mediaserver.py rename to ansible/molecule/default/tests/test_mediaserver.py diff --git a/molecule/default/tests/test_mediavault.py b/ansible/molecule/default/tests/test_mediavault.py similarity index 100% rename from molecule/default/tests/test_mediavault.py rename to ansible/molecule/default/tests/test_mediavault.py diff --git a/molecule/default/tests/test_mediaworker.py b/ansible/molecule/default/tests/test_mediaworker.py similarity index 100% rename from molecule/default/tests/test_mediaworker.py rename to ansible/molecule/default/tests/test_mediaworker.py diff --git a/molecule/default/tests/test_mirismanager.py b/ansible/molecule/default/tests/test_mirismanager.py similarity index 100% rename from molecule/default/tests/test_mirismanager.py rename to ansible/molecule/default/tests/test_mirismanager.py diff --git a/molecule/default/tests/test_msmonitor.py b/ansible/molecule/default/tests/test_msmonitor.py similarity index 100% rename from molecule/default/tests/test_msmonitor.py rename to ansible/molecule/default/tests/test_msmonitor.py diff --git a/molecule/default/tests/test_netcapture.py b/ansible/molecule/default/tests/test_netcapture.py similarity index 100% rename from molecule/default/tests/test_netcapture.py rename to ansible/molecule/default/tests/test_netcapture.py diff --git a/molecule/default/tests/test_nginx.py b/ansible/molecule/default/tests/test_nginx.py similarity index 100% rename from molecule/default/tests/test_nginx.py rename to ansible/molecule/default/tests/test_nginx.py diff --git a/molecule/default/tests/test_ntp.py b/ansible/molecule/default/tests/test_ntp.py similarity index 100% rename from molecule/default/tests/test_ntp.py rename to ansible/molecule/default/tests/test_ntp.py diff --git a/molecule/default/tests/test_postfix.py b/ansible/molecule/default/tests/test_postfix.py similarity index 100% rename from molecule/default/tests/test_postfix.py rename to ansible/molecule/default/tests/test_postfix.py diff --git a/molecule/default/tests/test_postgres.py b/ansible/molecule/default/tests/test_postgres.py similarity index 100% rename from molecule/default/tests/test_postgres.py rename to ansible/molecule/default/tests/test_postgres.py diff --git a/molecule/default/tests/test_python3.py b/ansible/molecule/default/tests/test_python3.py similarity index 100% rename from molecule/default/tests/test_python3.py rename to ansible/molecule/default/tests/test_python3.py diff --git a/playbooks/base.yml b/ansible/playbooks/base.yml similarity index 100% rename from playbooks/base.yml rename to ansible/playbooks/base.yml diff --git a/playbooks/bench.yml b/ansible/playbooks/bench.yml similarity index 100% rename from playbooks/bench.yml rename to ansible/playbooks/bench.yml diff --git a/playbooks/celerity.yml b/ansible/playbooks/celerity.yml similarity index 100% rename from playbooks/celerity.yml rename to ansible/playbooks/celerity.yml diff --git a/playbooks/cluster.yml b/ansible/playbooks/cluster.yml similarity index 100% rename from playbooks/cluster.yml rename to ansible/playbooks/cluster.yml diff --git a/playbooks/letsencrypt.yml b/ansible/playbooks/letsencrypt.yml similarity index 100% rename from playbooks/letsencrypt.yml rename to ansible/playbooks/letsencrypt.yml diff --git a/playbooks/mediaimport.yml b/ansible/playbooks/mediaimport.yml similarity index 100% rename from playbooks/mediaimport.yml rename to ansible/playbooks/mediaimport.yml diff --git a/playbooks/mediaserver.yml b/ansible/playbooks/mediaserver.yml similarity index 100% rename from playbooks/mediaserver.yml rename to ansible/playbooks/mediaserver.yml diff --git a/playbooks/mediavault.yml b/ansible/playbooks/mediavault.yml similarity index 100% rename from playbooks/mediavault.yml rename to ansible/playbooks/mediavault.yml diff --git a/playbooks/mediaworker.yml b/ansible/playbooks/mediaworker.yml similarity index 100% rename from playbooks/mediaworker.yml rename to ansible/playbooks/mediaworker.yml diff --git a/playbooks/migrate-debian.yml b/ansible/playbooks/migrate-debian.yml similarity index 100% rename from playbooks/migrate-debian.yml rename to ansible/playbooks/migrate-debian.yml diff --git a/playbooks/mirismanager.yml b/ansible/playbooks/mirismanager.yml similarity index 100% rename from playbooks/mirismanager.yml rename to ansible/playbooks/mirismanager.yml diff --git a/playbooks/msmonitor.yml b/ansible/playbooks/msmonitor.yml similarity index 100% rename from playbooks/msmonitor.yml rename to ansible/playbooks/msmonitor.yml diff --git a/playbooks/netcapture.yml b/ansible/playbooks/netcapture.yml similarity index 100% rename from playbooks/netcapture.yml rename to ansible/playbooks/netcapture.yml diff --git a/playbooks/postfix.yml b/ansible/playbooks/postfix.yml similarity index 100% rename from playbooks/postfix.yml rename to ansible/playbooks/postfix.yml diff --git a/playbooks/postgres-ha.yml b/ansible/playbooks/postgres-ha.yml similarity index 100% rename from playbooks/postgres-ha.yml rename to ansible/playbooks/postgres-ha.yml diff --git a/playbooks/postgres.yml b/ansible/playbooks/postgres.yml similarity index 100% rename from playbooks/postgres.yml rename to ansible/playbooks/postgres.yml diff --git a/playbooks/repos.yml b/ansible/playbooks/repos.yml similarity index 100% rename from playbooks/repos.yml rename to ansible/playbooks/repos.yml diff --git a/site.yml b/ansible/playbooks/site.yml similarity index 51% rename from site.yml rename to ansible/playbooks/site.yml index 7cb88ce9..e86b5c07 100755 --- a/site.yml +++ b/ansible/playbooks/site.yml @@ -10,25 +10,25 @@ changed_when: "'es_pyinstall' in python_install.stdout_lines" raw: command -v python3 || echo es_pyinstall && apt update && apt install -y python3-minimal python3-apt tags: always -- import_playbook: "playbooks/{{ 'postgres-ha' if groups['postgres']|d('') | length > 1 else 'postgres' }}.yml" +- import_playbook: "{{ 'postgres-ha' if groups['postgres']|d('') | length > 1 else 'postgres' }}.yml" tags: postgres -- import_playbook: playbooks/msmonitor.yml +- import_playbook: msmonitor.yml tags: monitor -- import_playbook: playbooks/mirismanager.yml +- import_playbook: mirismanager.yml tags: manager -- import_playbook: playbooks/wowza.yml +- import_playbook: wowza.yml tags: wowza -- import_playbook: playbooks/celerity.yml +- import_playbook: celerity.yml tags: celerity -- import_playbook: playbooks/mediaworker.yml +- import_playbook: mediaworker.yml tags: worker -- import_playbook: playbooks/mediaserver.yml +- import_playbook: mediaserver.yml tags: server -- import_playbook: playbooks/mediavault.yml +- import_playbook: mediavault.yml tags: vault -- import_playbook: playbooks/mediaimport.yml +- import_playbook: mediaimport.yml tags: import -- import_playbook: playbooks/netcapture.yml +- import_playbook: netcapture.yml tags: netcapture ... diff --git a/playbooks/tests.yml b/ansible/playbooks/tests.yml similarity index 100% rename from playbooks/tests.yml rename to ansible/playbooks/tests.yml diff --git a/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml similarity index 100% rename from playbooks/upgrade.yml rename to ansible/playbooks/upgrade.yml diff --git a/playbooks/users.yml b/ansible/playbooks/users.yml similarity index 100% rename from playbooks/users.yml rename to ansible/playbooks/users.yml diff --git a/playbooks/wowza-ha.yml b/ansible/playbooks/wowza-ha.yml similarity index 100% rename from playbooks/wowza-ha.yml rename to ansible/playbooks/wowza-ha.yml diff --git a/playbooks/wowza.yml b/ansible/playbooks/wowza.yml similarity index 100% rename from playbooks/wowza.yml rename to ansible/playbooks/wowza.yml diff --git a/plugins/action/source_file.py b/ansible/plugins/action/source_file.py similarity index 100% rename from plugins/action/source_file.py rename to ansible/plugins/action/source_file.py diff --git a/requirements.dev.in b/ansible/requirements.dev.in similarity index 100% rename from requirements.dev.in rename to ansible/requirements.dev.in diff --git a/requirements.dev.txt b/ansible/requirements.dev.txt similarity index 100% rename from requirements.dev.txt rename to ansible/requirements.dev.txt diff --git a/requirements.in b/ansible/requirements.in similarity index 100% rename from requirements.in rename to ansible/requirements.in diff --git a/requirements.txt b/ansible/requirements.txt similarity index 100% rename from requirements.txt rename to ansible/requirements.txt diff --git a/requirements.yml b/ansible/requirements.yml similarity index 100% rename from requirements.yml rename to ansible/requirements.yml diff --git a/roles/base/meta/main.yml b/ansible/roles/base/meta/main.yml similarity index 100% rename from roles/base/meta/main.yml rename to ansible/roles/base/meta/main.yml diff --git a/roles/bench-server/defaults/main.yml b/ansible/roles/bench-server/defaults/main.yml similarity index 100% rename from roles/bench-server/defaults/main.yml rename to ansible/roles/bench-server/defaults/main.yml diff --git a/roles/bench-server/meta/main.yml b/ansible/roles/bench-server/meta/main.yml similarity index 100% rename from roles/bench-server/meta/main.yml rename to ansible/roles/bench-server/meta/main.yml diff --git a/roles/bench-server/tasks/main.yml b/ansible/roles/bench-server/tasks/main.yml similarity index 100% rename from roles/bench-server/tasks/main.yml rename to ansible/roles/bench-server/tasks/main.yml diff --git a/roles/bench-server/templates/bench-streaming.conf.j2 b/ansible/roles/bench-server/templates/bench-streaming.conf.j2 similarity index 100% rename from roles/bench-server/templates/bench-streaming.conf.j2 rename to ansible/roles/bench-server/templates/bench-streaming.conf.j2 diff --git a/roles/bench-worker/defaults/main.yml b/ansible/roles/bench-worker/defaults/main.yml similarity index 100% rename from roles/bench-worker/defaults/main.yml rename to ansible/roles/bench-worker/defaults/main.yml diff --git a/roles/bench-worker/meta/main.yml b/ansible/roles/bench-worker/meta/main.yml similarity index 100% rename from roles/bench-worker/meta/main.yml rename to ansible/roles/bench-worker/meta/main.yml diff --git a/roles/bench-worker/tasks/main.yml b/ansible/roles/bench-worker/tasks/main.yml similarity index 100% rename from roles/bench-worker/tasks/main.yml rename to ansible/roles/bench-worker/tasks/main.yml diff --git a/roles/celerity/defaults/main.yml b/ansible/roles/celerity/defaults/main.yml similarity index 100% rename from roles/celerity/defaults/main.yml rename to ansible/roles/celerity/defaults/main.yml diff --git a/roles/celerity/handlers/main.yml b/ansible/roles/celerity/handlers/main.yml similarity index 100% rename from roles/celerity/handlers/main.yml rename to ansible/roles/celerity/handlers/main.yml diff --git a/roles/celerity/meta/main.yml b/ansible/roles/celerity/meta/main.yml similarity index 100% rename from roles/celerity/meta/main.yml rename to ansible/roles/celerity/meta/main.yml diff --git a/roles/celerity/tasks/main.yml b/ansible/roles/celerity/tasks/main.yml similarity index 100% rename from roles/celerity/tasks/main.yml rename to ansible/roles/celerity/tasks/main.yml diff --git a/roles/celerity/templates/celerity-config.py.j2 b/ansible/roles/celerity/templates/celerity-config.py.j2 similarity index 100% rename from roles/celerity/templates/celerity-config.py.j2 rename to ansible/roles/celerity/templates/celerity-config.py.j2 diff --git a/roles/ceph-rbd/defaults/main.yml b/ansible/roles/ceph-rbd/defaults/main.yml similarity index 100% rename from roles/ceph-rbd/defaults/main.yml rename to ansible/roles/ceph-rbd/defaults/main.yml diff --git a/roles/ceph-rbd/handlers/main.yml b/ansible/roles/ceph-rbd/handlers/main.yml similarity index 100% rename from roles/ceph-rbd/handlers/main.yml rename to ansible/roles/ceph-rbd/handlers/main.yml diff --git a/roles/ceph-rbd/tasks/main.yml b/ansible/roles/ceph-rbd/tasks/main.yml similarity index 100% rename from roles/ceph-rbd/tasks/main.yml rename to ansible/roles/ceph-rbd/tasks/main.yml diff --git a/roles/ceph-rbd/templates/ceph.client.user.keyring.j2 b/ansible/roles/ceph-rbd/templates/ceph.client.user.keyring.j2 similarity index 100% rename from roles/ceph-rbd/templates/ceph.client.user.keyring.j2 rename to ansible/roles/ceph-rbd/templates/ceph.client.user.keyring.j2 diff --git a/roles/ceph-rbd/templates/ceph.conf.j2 b/ansible/roles/ceph-rbd/templates/ceph.conf.j2 similarity index 100% rename from roles/ceph-rbd/templates/ceph.conf.j2 rename to ansible/roles/ceph-rbd/templates/ceph.conf.j2 diff --git a/roles/cluster/defaults/main.yml b/ansible/roles/cluster/defaults/main.yml similarity index 100% rename from roles/cluster/defaults/main.yml rename to ansible/roles/cluster/defaults/main.yml diff --git a/roles/cluster/handlers/main.yml b/ansible/roles/cluster/handlers/main.yml similarity index 100% rename from roles/cluster/handlers/main.yml rename to ansible/roles/cluster/handlers/main.yml diff --git a/roles/cluster/tasks/main.yml b/ansible/roles/cluster/tasks/main.yml similarity index 100% rename from roles/cluster/tasks/main.yml rename to ansible/roles/cluster/tasks/main.yml diff --git a/roles/cluster/templates/corosync.conf.j2 b/ansible/roles/cluster/templates/corosync.conf.j2 similarity index 100% rename from roles/cluster/templates/corosync.conf.j2 rename to ansible/roles/cluster/templates/corosync.conf.j2 diff --git a/roles/conf/defaults/main.yml b/ansible/roles/conf/defaults/main.yml similarity index 100% rename from roles/conf/defaults/main.yml rename to ansible/roles/conf/defaults/main.yml diff --git a/roles/conf/tasks/main.yml b/ansible/roles/conf/tasks/main.yml similarity index 100% rename from roles/conf/tasks/main.yml rename to ansible/roles/conf/tasks/main.yml diff --git a/roles/elastic/defaults/main.yml b/ansible/roles/elastic/defaults/main.yml similarity index 100% rename from roles/elastic/defaults/main.yml rename to ansible/roles/elastic/defaults/main.yml diff --git a/roles/elastic/handlers/main.yml b/ansible/roles/elastic/handlers/main.yml similarity index 100% rename from roles/elastic/handlers/main.yml rename to ansible/roles/elastic/handlers/main.yml diff --git a/roles/elastic/tasks/main.yml b/ansible/roles/elastic/tasks/main.yml similarity index 100% rename from roles/elastic/tasks/main.yml rename to ansible/roles/elastic/tasks/main.yml diff --git a/roles/elastic/templates/apm-server.yml.j2 b/ansible/roles/elastic/templates/apm-server.yml.j2 similarity index 100% rename from roles/elastic/templates/apm-server.yml.j2 rename to ansible/roles/elastic/templates/apm-server.yml.j2 diff --git a/roles/elastic/templates/kibana.yml.j2 b/ansible/roles/elastic/templates/kibana.yml.j2 similarity index 100% rename from roles/elastic/templates/kibana.yml.j2 rename to ansible/roles/elastic/templates/kibana.yml.j2 diff --git a/roles/fail2ban/defaults/main.yml b/ansible/roles/fail2ban/defaults/main.yml similarity index 100% rename from roles/fail2ban/defaults/main.yml rename to ansible/roles/fail2ban/defaults/main.yml diff --git a/roles/fail2ban/handlers/main.yml b/ansible/roles/fail2ban/handlers/main.yml similarity index 100% rename from roles/fail2ban/handlers/main.yml rename to ansible/roles/fail2ban/handlers/main.yml diff --git a/roles/fail2ban/tasks/main.yml b/ansible/roles/fail2ban/tasks/main.yml similarity index 100% rename from roles/fail2ban/tasks/main.yml rename to ansible/roles/fail2ban/tasks/main.yml diff --git a/roles/fail2ban/templates/jail.local.j2 b/ansible/roles/fail2ban/templates/jail.local.j2 similarity index 100% rename from roles/fail2ban/templates/jail.local.j2 rename to ansible/roles/fail2ban/templates/jail.local.j2 diff --git a/roles/ferm-configure/defaults/main.yml b/ansible/roles/ferm-configure/defaults/main.yml similarity index 100% rename from roles/ferm-configure/defaults/main.yml rename to ansible/roles/ferm-configure/defaults/main.yml diff --git a/roles/ferm-configure/handlers/main.yml b/ansible/roles/ferm-configure/handlers/main.yml similarity index 100% rename from roles/ferm-configure/handlers/main.yml rename to ansible/roles/ferm-configure/handlers/main.yml diff --git a/roles/ferm-configure/tasks/main.yml b/ansible/roles/ferm-configure/tasks/main.yml similarity index 100% rename from roles/ferm-configure/tasks/main.yml rename to ansible/roles/ferm-configure/tasks/main.yml diff --git a/roles/ferm-configure/templates/ferm_rules_forward.conf.j2 b/ansible/roles/ferm-configure/templates/ferm_rules_forward.conf.j2 similarity index 100% rename from roles/ferm-configure/templates/ferm_rules_forward.conf.j2 rename to ansible/roles/ferm-configure/templates/ferm_rules_forward.conf.j2 diff --git a/roles/ferm-configure/templates/ferm_rules_input.conf.j2 b/ansible/roles/ferm-configure/templates/ferm_rules_input.conf.j2 similarity index 100% rename from roles/ferm-configure/templates/ferm_rules_input.conf.j2 rename to ansible/roles/ferm-configure/templates/ferm_rules_input.conf.j2 diff --git a/roles/ferm-configure/templates/ferm_rules_output.conf.j2 b/ansible/roles/ferm-configure/templates/ferm_rules_output.conf.j2 similarity index 100% rename from roles/ferm-configure/templates/ferm_rules_output.conf.j2 rename to ansible/roles/ferm-configure/templates/ferm_rules_output.conf.j2 diff --git a/roles/ferm-install/defaults/main.yml b/ansible/roles/ferm-install/defaults/main.yml similarity index 100% rename from roles/ferm-install/defaults/main.yml rename to ansible/roles/ferm-install/defaults/main.yml diff --git a/roles/ferm-install/handlers/main.yml b/ansible/roles/ferm-install/handlers/main.yml similarity index 100% rename from roles/ferm-install/handlers/main.yml rename to ansible/roles/ferm-install/handlers/main.yml diff --git a/roles/ferm-install/tasks/main.yml b/ansible/roles/ferm-install/tasks/main.yml similarity index 100% rename from roles/ferm-install/tasks/main.yml rename to ansible/roles/ferm-install/tasks/main.yml diff --git a/roles/ferm-install/templates/ferm.conf.j2 b/ansible/roles/ferm-install/templates/ferm.conf.j2 similarity index 100% rename from roles/ferm-install/templates/ferm.conf.j2 rename to ansible/roles/ferm-install/templates/ferm.conf.j2 diff --git a/roles/haproxy/defaults/main.yml b/ansible/roles/haproxy/defaults/main.yml similarity index 100% rename from roles/haproxy/defaults/main.yml rename to ansible/roles/haproxy/defaults/main.yml diff --git a/roles/haproxy/handlers/main.yml b/ansible/roles/haproxy/handlers/main.yml similarity index 100% rename from roles/haproxy/handlers/main.yml rename to ansible/roles/haproxy/handlers/main.yml diff --git a/roles/haproxy/tasks/main.yml b/ansible/roles/haproxy/tasks/main.yml similarity index 100% rename from roles/haproxy/tasks/main.yml rename to ansible/roles/haproxy/tasks/main.yml diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 similarity index 100% rename from roles/haproxy/templates/haproxy.cfg.j2 rename to ansible/roles/haproxy/templates/haproxy.cfg.j2 diff --git a/roles/init/defaults/main.yml b/ansible/roles/init/defaults/main.yml similarity index 100% rename from roles/init/defaults/main.yml rename to ansible/roles/init/defaults/main.yml diff --git a/roles/init/tasks/main.yml b/ansible/roles/init/tasks/main.yml similarity index 100% rename from roles/init/tasks/main.yml rename to ansible/roles/init/tasks/main.yml diff --git a/roles/letsencrypt/defaults/main.yml b/ansible/roles/letsencrypt/defaults/main.yml similarity index 100% rename from roles/letsencrypt/defaults/main.yml rename to ansible/roles/letsencrypt/defaults/main.yml diff --git a/roles/letsencrypt/handlers/main.yml b/ansible/roles/letsencrypt/handlers/main.yml similarity index 100% rename from roles/letsencrypt/handlers/main.yml rename to ansible/roles/letsencrypt/handlers/main.yml diff --git a/roles/letsencrypt/tasks/main.yml b/ansible/roles/letsencrypt/tasks/main.yml similarity index 100% rename from roles/letsencrypt/tasks/main.yml rename to ansible/roles/letsencrypt/tasks/main.yml diff --git a/roles/mediaimport/defaults/main.yml b/ansible/roles/mediaimport/defaults/main.yml similarity index 100% rename from roles/mediaimport/defaults/main.yml rename to ansible/roles/mediaimport/defaults/main.yml diff --git a/roles/mediaimport/files/mediaimport b/ansible/roles/mediaimport/files/mediaimport similarity index 100% rename from roles/mediaimport/files/mediaimport rename to ansible/roles/mediaimport/files/mediaimport diff --git a/roles/mediaimport/files/mediaimport.py b/ansible/roles/mediaimport/files/mediaimport.py similarity index 100% rename from roles/mediaimport/files/mediaimport.py rename to ansible/roles/mediaimport/files/mediaimport.py diff --git a/roles/mediaimport/files/on-upload b/ansible/roles/mediaimport/files/on-upload similarity index 100% rename from roles/mediaimport/files/on-upload rename to ansible/roles/mediaimport/files/on-upload diff --git a/roles/mediaimport/files/on-upload.go b/ansible/roles/mediaimport/files/on-upload.go similarity index 100% rename from roles/mediaimport/files/on-upload.go rename to ansible/roles/mediaimport/files/on-upload.go diff --git a/roles/mediaimport/handlers/main.yml b/ansible/roles/mediaimport/handlers/main.yml similarity index 100% rename from roles/mediaimport/handlers/main.yml rename to ansible/roles/mediaimport/handlers/main.yml diff --git a/roles/mediaimport/meta/main.yml b/ansible/roles/mediaimport/meta/main.yml similarity index 100% rename from roles/mediaimport/meta/main.yml rename to ansible/roles/mediaimport/meta/main.yml diff --git a/roles/mediaimport/tasks/main.yml b/ansible/roles/mediaimport/tasks/main.yml similarity index 100% rename from roles/mediaimport/tasks/main.yml rename to ansible/roles/mediaimport/tasks/main.yml diff --git a/roles/mediaimport/templates/mediaimport.json.j2 b/ansible/roles/mediaimport/templates/mediaimport.json.j2 similarity index 100% rename from roles/mediaimport/templates/mediaimport.json.j2 rename to ansible/roles/mediaimport/templates/mediaimport.json.j2 diff --git a/roles/mediaimport/templates/sftp_config.j2 b/ansible/roles/mediaimport/templates/sftp_config.j2 similarity index 100% rename from roles/mediaimport/templates/sftp_config.j2 rename to ansible/roles/mediaimport/templates/sftp_config.j2 diff --git a/roles/mediaserver/defaults/main.yml b/ansible/roles/mediaserver/defaults/main.yml similarity index 100% rename from roles/mediaserver/defaults/main.yml rename to ansible/roles/mediaserver/defaults/main.yml diff --git a/roles/mediaserver/handlers/main.yml b/ansible/roles/mediaserver/handlers/main.yml similarity index 100% rename from roles/mediaserver/handlers/main.yml rename to ansible/roles/mediaserver/handlers/main.yml diff --git a/roles/mediaserver/meta/main.yml b/ansible/roles/mediaserver/meta/main.yml similarity index 100% rename from roles/mediaserver/meta/main.yml rename to ansible/roles/mediaserver/meta/main.yml diff --git a/roles/mediaserver/tasks/main.yml b/ansible/roles/mediaserver/tasks/main.yml similarity index 100% rename from roles/mediaserver/tasks/main.yml rename to ansible/roles/mediaserver/tasks/main.yml diff --git a/roles/mediaserver/templates/celerity-config.py.j2 b/ansible/roles/mediaserver/templates/celerity-config.py.j2 similarity index 100% rename from roles/mediaserver/templates/celerity-config.py.j2 rename to ansible/roles/mediaserver/templates/celerity-config.py.j2 diff --git a/roles/mediavault/defaults/main.yml b/ansible/roles/mediavault/defaults/main.yml similarity index 100% rename from roles/mediavault/defaults/main.yml rename to ansible/roles/mediavault/defaults/main.yml diff --git a/roles/mediavault/handlers/main.yml b/ansible/roles/mediavault/handlers/main.yml similarity index 100% rename from roles/mediavault/handlers/main.yml rename to ansible/roles/mediavault/handlers/main.yml diff --git a/roles/mediavault/meta/main.yml b/ansible/roles/mediavault/meta/main.yml similarity index 100% rename from roles/mediavault/meta/main.yml rename to ansible/roles/mediavault/meta/main.yml diff --git a/roles/mediavault/tasks/main.yml b/ansible/roles/mediavault/tasks/main.yml similarity index 100% rename from roles/mediavault/tasks/main.yml rename to ansible/roles/mediavault/tasks/main.yml diff --git a/roles/mediavault/templates/systemd-backup-service.j2 b/ansible/roles/mediavault/templates/systemd-backup-service.j2 similarity index 100% rename from roles/mediavault/templates/systemd-backup-service.j2 rename to ansible/roles/mediavault/templates/systemd-backup-service.j2 diff --git a/roles/mediavault/templates/systemd-backup-timer.j2 b/ansible/roles/mediavault/templates/systemd-backup-timer.j2 similarity index 100% rename from roles/mediavault/templates/systemd-backup-timer.j2 rename to ansible/roles/mediavault/templates/systemd-backup-timer.j2 diff --git a/roles/mediavault/templates/systemd-mailer-script.j2 b/ansible/roles/mediavault/templates/systemd-mailer-script.j2 similarity index 100% rename from roles/mediavault/templates/systemd-mailer-script.j2 rename to ansible/roles/mediavault/templates/systemd-mailer-script.j2 diff --git a/roles/mediavault/templates/systemd-mailer-service.j2 b/ansible/roles/mediavault/templates/systemd-mailer-service.j2 similarity index 100% rename from roles/mediavault/templates/systemd-mailer-service.j2 rename to ansible/roles/mediavault/templates/systemd-mailer-service.j2 diff --git a/roles/mediaworker/defaults/main.yml b/ansible/roles/mediaworker/defaults/main.yml similarity index 100% rename from roles/mediaworker/defaults/main.yml rename to ansible/roles/mediaworker/defaults/main.yml diff --git a/roles/mediaworker/handlers/main.yml b/ansible/roles/mediaworker/handlers/main.yml similarity index 100% rename from roles/mediaworker/handlers/main.yml rename to ansible/roles/mediaworker/handlers/main.yml diff --git a/roles/mediaworker/meta/main.yml b/ansible/roles/mediaworker/meta/main.yml similarity index 100% rename from roles/mediaworker/meta/main.yml rename to ansible/roles/mediaworker/meta/main.yml diff --git a/roles/mediaworker/tasks/main.yml b/ansible/roles/mediaworker/tasks/main.yml similarity index 100% rename from roles/mediaworker/tasks/main.yml rename to ansible/roles/mediaworker/tasks/main.yml diff --git a/roles/mediaworker/templates/celerity-config.py.j2 b/ansible/roles/mediaworker/templates/celerity-config.py.j2 similarity index 100% rename from roles/mediaworker/templates/celerity-config.py.j2 rename to ansible/roles/mediaworker/templates/celerity-config.py.j2 diff --git a/roles/metricbeat/defaults/main.yml b/ansible/roles/metricbeat/defaults/main.yml similarity index 100% rename from roles/metricbeat/defaults/main.yml rename to ansible/roles/metricbeat/defaults/main.yml diff --git a/roles/metricbeat/handlers/main.yml b/ansible/roles/metricbeat/handlers/main.yml similarity index 100% rename from roles/metricbeat/handlers/main.yml rename to ansible/roles/metricbeat/handlers/main.yml diff --git a/roles/metricbeat/tasks/main.yml b/ansible/roles/metricbeat/tasks/main.yml similarity index 100% rename from roles/metricbeat/tasks/main.yml rename to ansible/roles/metricbeat/tasks/main.yml diff --git a/roles/metricbeat/templates/metricbeat.yml.j2 b/ansible/roles/metricbeat/templates/metricbeat.yml.j2 similarity index 100% rename from roles/metricbeat/templates/metricbeat.yml.j2 rename to ansible/roles/metricbeat/templates/metricbeat.yml.j2 diff --git a/roles/metricbeat/templates/postgresql.yml.j2 b/ansible/roles/metricbeat/templates/postgresql.yml.j2 similarity index 100% rename from roles/metricbeat/templates/postgresql.yml.j2 rename to ansible/roles/metricbeat/templates/postgresql.yml.j2 diff --git a/roles/mirismanager/defaults/main.yml b/ansible/roles/mirismanager/defaults/main.yml similarity index 100% rename from roles/mirismanager/defaults/main.yml rename to ansible/roles/mirismanager/defaults/main.yml diff --git a/roles/mirismanager/files/set_site_url.py b/ansible/roles/mirismanager/files/set_site_url.py similarity index 100% rename from roles/mirismanager/files/set_site_url.py rename to ansible/roles/mirismanager/files/set_site_url.py diff --git a/roles/mirismanager/handlers/main.yml b/ansible/roles/mirismanager/handlers/main.yml similarity index 100% rename from roles/mirismanager/handlers/main.yml rename to ansible/roles/mirismanager/handlers/main.yml diff --git a/roles/mirismanager/meta/main.yml b/ansible/roles/mirismanager/meta/main.yml similarity index 100% rename from roles/mirismanager/meta/main.yml rename to ansible/roles/mirismanager/meta/main.yml diff --git a/roles/mirismanager/tasks/main.yml b/ansible/roles/mirismanager/tasks/main.yml similarity index 100% rename from roles/mirismanager/tasks/main.yml rename to ansible/roles/mirismanager/tasks/main.yml diff --git a/roles/msmonitor/defaults/main.yml b/ansible/roles/msmonitor/defaults/main.yml similarity index 100% rename from roles/msmonitor/defaults/main.yml rename to ansible/roles/msmonitor/defaults/main.yml diff --git a/roles/msmonitor/handlers/main.yml b/ansible/roles/msmonitor/handlers/main.yml similarity index 100% rename from roles/msmonitor/handlers/main.yml rename to ansible/roles/msmonitor/handlers/main.yml diff --git a/roles/msmonitor/meta/main.yml b/ansible/roles/msmonitor/meta/main.yml similarity index 100% rename from roles/msmonitor/meta/main.yml rename to ansible/roles/msmonitor/meta/main.yml diff --git a/roles/msmonitor/tasks/main.yml b/ansible/roles/msmonitor/tasks/main.yml similarity index 100% rename from roles/msmonitor/tasks/main.yml rename to ansible/roles/msmonitor/tasks/main.yml diff --git a/roles/netcapture/defaults/main.yml b/ansible/roles/netcapture/defaults/main.yml similarity index 100% rename from roles/netcapture/defaults/main.yml rename to ansible/roles/netcapture/defaults/main.yml diff --git a/roles/netcapture/meta/main.yml b/ansible/roles/netcapture/meta/main.yml similarity index 100% rename from roles/netcapture/meta/main.yml rename to ansible/roles/netcapture/meta/main.yml diff --git a/roles/netcapture/tasks/main.yml b/ansible/roles/netcapture/tasks/main.yml similarity index 100% rename from roles/netcapture/tasks/main.yml rename to ansible/roles/netcapture/tasks/main.yml diff --git a/roles/netcapture/templates/miris-api.json.j2 b/ansible/roles/netcapture/templates/miris-api.json.j2 similarity index 100% rename from roles/netcapture/templates/miris-api.json.j2 rename to ansible/roles/netcapture/templates/miris-api.json.j2 diff --git a/roles/netcapture/templates/netcapture.json.j2 b/ansible/roles/netcapture/templates/netcapture.json.j2 similarity index 100% rename from roles/netcapture/templates/netcapture.json.j2 rename to ansible/roles/netcapture/templates/netcapture.json.j2 diff --git a/roles/network/defaults/main.yml b/ansible/roles/network/defaults/main.yml similarity index 100% rename from roles/network/defaults/main.yml rename to ansible/roles/network/defaults/main.yml diff --git a/roles/network/tasks/main.yml b/ansible/roles/network/tasks/main.yml similarity index 100% rename from roles/network/tasks/main.yml rename to ansible/roles/network/tasks/main.yml diff --git a/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml similarity index 100% rename from roles/nginx/defaults/main.yml rename to ansible/roles/nginx/defaults/main.yml diff --git a/roles/nginx/handlers/main.yml b/ansible/roles/nginx/handlers/main.yml similarity index 100% rename from roles/nginx/handlers/main.yml rename to ansible/roles/nginx/handlers/main.yml diff --git a/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml similarity index 100% rename from roles/nginx/tasks/main.yml rename to ansible/roles/nginx/tasks/main.yml diff --git a/roles/ocfs2/defaults/main.yml b/ansible/roles/ocfs2/defaults/main.yml similarity index 100% rename from roles/ocfs2/defaults/main.yml rename to ansible/roles/ocfs2/defaults/main.yml diff --git a/roles/ocfs2/handlers/main.yml b/ansible/roles/ocfs2/handlers/main.yml similarity index 100% rename from roles/ocfs2/handlers/main.yml rename to ansible/roles/ocfs2/handlers/main.yml diff --git a/roles/ocfs2/tasks/main.yml b/ansible/roles/ocfs2/tasks/main.yml similarity index 100% rename from roles/ocfs2/tasks/main.yml rename to ansible/roles/ocfs2/tasks/main.yml diff --git a/roles/ocfs2/templates/cluster.conf.j2 b/ansible/roles/ocfs2/templates/cluster.conf.j2 similarity index 100% rename from roles/ocfs2/templates/cluster.conf.j2 rename to ansible/roles/ocfs2/templates/cluster.conf.j2 diff --git a/roles/postfix/defaults/main.yml b/ansible/roles/postfix/defaults/main.yml similarity index 100% rename from roles/postfix/defaults/main.yml rename to ansible/roles/postfix/defaults/main.yml diff --git a/roles/postfix/handlers/main.yml b/ansible/roles/postfix/handlers/main.yml similarity index 100% rename from roles/postfix/handlers/main.yml rename to ansible/roles/postfix/handlers/main.yml diff --git a/roles/postfix/meta/main.yml b/ansible/roles/postfix/meta/main.yml similarity index 100% rename from roles/postfix/meta/main.yml rename to ansible/roles/postfix/meta/main.yml diff --git a/roles/postfix/tasks/main.yml b/ansible/roles/postfix/tasks/main.yml similarity index 100% rename from roles/postfix/tasks/main.yml rename to ansible/roles/postfix/tasks/main.yml diff --git a/roles/postfix/templates/main.cf.j2 b/ansible/roles/postfix/templates/main.cf.j2 similarity index 100% rename from roles/postfix/templates/main.cf.j2 rename to ansible/roles/postfix/templates/main.cf.j2 diff --git a/roles/postgres-ha/defaults/main.yml b/ansible/roles/postgres-ha/defaults/main.yml similarity index 100% rename from roles/postgres-ha/defaults/main.yml rename to ansible/roles/postgres-ha/defaults/main.yml diff --git a/roles/postgres-ha/handlers/main.yml b/ansible/roles/postgres-ha/handlers/main.yml similarity index 100% rename from roles/postgres-ha/handlers/main.yml rename to ansible/roles/postgres-ha/handlers/main.yml diff --git a/roles/postgres-ha/tasks/main.yml b/ansible/roles/postgres-ha/tasks/main.yml similarity index 100% rename from roles/postgres-ha/tasks/main.yml rename to ansible/roles/postgres-ha/tasks/main.yml diff --git a/roles/postgres-ha/templates/rephacheck.conf.j2 b/ansible/roles/postgres-ha/templates/rephacheck.conf.j2 similarity index 100% rename from roles/postgres-ha/templates/rephacheck.conf.j2 rename to ansible/roles/postgres-ha/templates/rephacheck.conf.j2 diff --git a/roles/postgres-ha/templates/rephacheck.py.j2 b/ansible/roles/postgres-ha/templates/rephacheck.py.j2 similarity index 100% rename from roles/postgres-ha/templates/rephacheck.py.j2 rename to ansible/roles/postgres-ha/templates/rephacheck.py.j2 diff --git a/roles/postgres-ha/templates/repmgr-event.py.j2 b/ansible/roles/postgres-ha/templates/repmgr-event.py.j2 similarity index 100% rename from roles/postgres-ha/templates/repmgr-event.py.j2 rename to ansible/roles/postgres-ha/templates/repmgr-event.py.j2 diff --git a/roles/postgres-ha/templates/repmgr.conf.j2 b/ansible/roles/postgres-ha/templates/repmgr.conf.j2 similarity index 100% rename from roles/postgres-ha/templates/repmgr.conf.j2 rename to ansible/roles/postgres-ha/templates/repmgr.conf.j2 diff --git a/roles/postgres/defaults/main.yml b/ansible/roles/postgres/defaults/main.yml similarity index 100% rename from roles/postgres/defaults/main.yml rename to ansible/roles/postgres/defaults/main.yml diff --git a/roles/postgres/handlers/main.yml b/ansible/roles/postgres/handlers/main.yml similarity index 100% rename from roles/postgres/handlers/main.yml rename to ansible/roles/postgres/handlers/main.yml diff --git a/roles/postgres/meta/main.yml b/ansible/roles/postgres/meta/main.yml similarity index 100% rename from roles/postgres/meta/main.yml rename to ansible/roles/postgres/meta/main.yml diff --git a/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml similarity index 100% rename from roles/postgres/tasks/main.yml rename to ansible/roles/postgres/tasks/main.yml diff --git a/roles/postgres/templates/pg_hba.conf.j2 b/ansible/roles/postgres/templates/pg_hba.conf.j2 similarity index 100% rename from roles/postgres/templates/pg_hba.conf.j2 rename to ansible/roles/postgres/templates/pg_hba.conf.j2 diff --git a/roles/proxy/defaults/main.yml b/ansible/roles/proxy/defaults/main.yml similarity index 100% rename from roles/proxy/defaults/main.yml rename to ansible/roles/proxy/defaults/main.yml diff --git a/roles/proxy/tasks/main.yml b/ansible/roles/proxy/tasks/main.yml similarity index 100% rename from roles/proxy/tasks/main.yml rename to ansible/roles/proxy/tasks/main.yml diff --git a/roles/sysconfig/defaults/main.yml b/ansible/roles/sysconfig/defaults/main.yml similarity index 100% rename from roles/sysconfig/defaults/main.yml rename to ansible/roles/sysconfig/defaults/main.yml diff --git a/roles/sysconfig/handlers/main.yml b/ansible/roles/sysconfig/handlers/main.yml similarity index 100% rename from roles/sysconfig/handlers/main.yml rename to ansible/roles/sysconfig/handlers/main.yml diff --git a/roles/sysconfig/tasks/locale.yml b/ansible/roles/sysconfig/tasks/locale.yml similarity index 100% rename from roles/sysconfig/tasks/locale.yml rename to ansible/roles/sysconfig/tasks/locale.yml diff --git a/roles/sysconfig/tasks/logs.yml b/ansible/roles/sysconfig/tasks/logs.yml similarity index 100% rename from roles/sysconfig/tasks/logs.yml rename to ansible/roles/sysconfig/tasks/logs.yml diff --git a/roles/sysconfig/tasks/main.yml b/ansible/roles/sysconfig/tasks/main.yml similarity index 100% rename from roles/sysconfig/tasks/main.yml rename to ansible/roles/sysconfig/tasks/main.yml diff --git a/roles/sysconfig/tasks/ntp.yml b/ansible/roles/sysconfig/tasks/ntp.yml similarity index 100% rename from roles/sysconfig/tasks/ntp.yml rename to ansible/roles/sysconfig/tasks/ntp.yml diff --git a/roles/sysconfig/tasks/repos.yml b/ansible/roles/sysconfig/tasks/repos.yml similarity index 100% rename from roles/sysconfig/tasks/repos.yml rename to ansible/roles/sysconfig/tasks/repos.yml diff --git a/roles/sysconfig/templates/ntp.conf.j2 b/ansible/roles/sysconfig/templates/ntp.conf.j2 similarity index 100% rename from roles/sysconfig/templates/ntp.conf.j2 rename to ansible/roles/sysconfig/templates/ntp.conf.j2 diff --git a/roles/users/defaults/main.yml b/ansible/roles/users/defaults/main.yml similarity index 100% rename from roles/users/defaults/main.yml rename to ansible/roles/users/defaults/main.yml diff --git a/roles/users/files/.bashrc b/ansible/roles/users/files/.bashrc similarity index 100% rename from roles/users/files/.bashrc rename to ansible/roles/users/files/.bashrc diff --git a/roles/users/files/.vimrc b/ansible/roles/users/files/.vimrc similarity index 100% rename from roles/users/files/.vimrc rename to ansible/roles/users/files/.vimrc diff --git a/roles/users/files/ubicast_support.pub b/ansible/roles/users/files/ubicast_support.pub similarity index 100% rename from roles/users/files/ubicast_support.pub rename to ansible/roles/users/files/ubicast_support.pub diff --git a/roles/users/handlers/main.yml b/ansible/roles/users/handlers/main.yml similarity index 100% rename from roles/users/handlers/main.yml rename to ansible/roles/users/handlers/main.yml diff --git a/roles/users/tasks/main.yml b/ansible/roles/users/tasks/main.yml similarity index 100% rename from roles/users/tasks/main.yml rename to ansible/roles/users/tasks/main.yml diff --git a/roles/wowza/defaults/main.yml b/ansible/roles/wowza/defaults/main.yml similarity index 100% rename from roles/wowza/defaults/main.yml rename to ansible/roles/wowza/defaults/main.yml diff --git a/roles/wowza/handlers/main.yml b/ansible/roles/wowza/handlers/main.yml similarity index 100% rename from roles/wowza/handlers/main.yml rename to ansible/roles/wowza/handlers/main.yml diff --git a/roles/wowza/meta/main.yml b/ansible/roles/wowza/meta/main.yml similarity index 100% rename from roles/wowza/meta/main.yml rename to ansible/roles/wowza/meta/main.yml diff --git a/roles/wowza/tasks/main.yml b/ansible/roles/wowza/tasks/main.yml similarity index 100% rename from roles/wowza/tasks/main.yml rename to ansible/roles/wowza/tasks/main.yml diff --git a/roles/wowza/templates/Server.xml.j2 b/ansible/roles/wowza/templates/Server.xml.j2 similarity index 100% rename from roles/wowza/templates/Server.xml.j2 rename to ansible/roles/wowza/templates/Server.xml.j2 diff --git a/roles/wowza/templates/Tune.xml.j2 b/ansible/roles/wowza/templates/Tune.xml.j2 similarity index 100% rename from roles/wowza/templates/Tune.xml.j2 rename to ansible/roles/wowza/templates/Tune.xml.j2 diff --git a/roles/wowza/templates/VHost.xml.j2 b/ansible/roles/wowza/templates/VHost.xml.j2 similarity index 100% rename from roles/wowza/templates/VHost.xml.j2 rename to ansible/roles/wowza/templates/VHost.xml.j2 diff --git a/roles/wowza/templates/live-application.xml.j2 b/ansible/roles/wowza/templates/live-application.xml.j2 similarity index 100% rename from roles/wowza/templates/live-application.xml.j2 rename to ansible/roles/wowza/templates/live-application.xml.j2 diff --git a/tester.py b/tester.py deleted file mode 100755 index 0ea4be2b..00000000 --- a/tester.py +++ /dev/null @@ -1,571 +0,0 @@ -#!/usr/bin/env python3 - -""" -Script to start tests and to manage their results -""" - -from io import StringIO -import base64 -import datetime -import glob -import os -import re -import socket -import subprocess -import sys -import time -import uuid - -import utils -from utils import log - -OUT_OF_SUPPORT_TEXT = """\033[93mWarning: -The system is out of support, UbiCast will not be notified if errors are detected. -Please contact UbiCast sales team (sales@ubicast.eu) to renew the support contract.\033[0m""" - - -class Logger(object): - def __init__(self, stream, log_buffer): - self.stream = stream - self.log_buffer = log_buffer - - def write(self, text): - self.stream.write(text) - self.stream.flush() - self.log_buffer.write(text) - self.log_buffer.flush() - - def flush(self): - pass - - -log_buffer = StringIO() -sys.stdout = Logger(sys.stdout, log_buffer) -sys.stderr = sys.stdout - - -def strip_colors(text): - return re.sub(r"\033\[[\d;]+m", "", text) - - -def escape(text): - html = text.strip() - html = html.replace("<", "<") - html = html.replace(">", ">") - html = html.replace("\033[90m", '<span style="color: gray;">') - html = html.replace("\033[91m", '<span style="color: red;">') - html = html.replace("\033[92m", '<span style="color: green;">') - html = html.replace("\033[93m", '<span style="color: orange;">') - html = html.replace("\033[94m", '<span style="color: blue;">') - html = html.replace("\033[95m", '<span style="color: purple;">') - html = strip_colors(html) - return html - - -def raid_idle(): - idle = True - devs = glob.glob("/sys/block/md*/md/sync_action") - for d in devs: - with open(d, "r") as f: - sync_state = f.read().strip() - if sync_state != "idle": - idle = False - print("State in %s is %s" % (d, sync_state)) - return idle - - -class Tester: - USAGE = ( - """%s [-e] [-f] [-b] [-n] [-p] [-d] [-h] [msuser] - -e: send email with report. - -f: send email with report only if at least one test failed. - -b: run only basic tests (exclude mediaserver tests). - -n: do not update envsetup repository. - -p: do not install packages. - -d: debug mode (can be started with non root users). - -h: show this message.""" - % __file__ - ) - VALID_ARGS = ["-e", "-f", "-b", "-n", "-p", "-d", "-h"] - MAX_LOG_FILES = 50 - NO_MAIL_FAILURES_COUNT = 5 - - def __init__(self, *args): - log("\033[96m-------------------------------\033[0m") - log("\033[96m- UbiCast applications tester -\033[0m") - log("\033[96m-------------------------------\033[0m") - args = list(args) - msuser = None - # Check if help is required - if "-h" in args: - log("USAGE: " + self.USAGE) - sys.exit(0) - for arg in args: - if arg.startswith("-"): - if arg not in self.VALID_ARGS: - log('Invalid argument given: "%s".\n' % arg) - log("USAGE: " + self.USAGE) - sys.exit(1) - else: - log("Optional target user: %s" % arg) - if not os.path.isdir(os.path.join("/home", arg)): - log("Mediaserver user %s does not exist" % arg) - sys.exit(1) - else: - msuser = arg - # Check current dir - root_dir = utils.get_dir(__file__) - if root_dir != "": - os.chdir(root_dir) - self.root_dir = root_dir - # Add to python path - if root_dir not in sys.path: - sys.path.append(root_dir) - # Check that this script is run by root - debug = "-d" in args - if os.getuid() != 0 and not debug: - log("This script should be run as root user.") - sys.exit(1) - # Update envsetup files - if "-n" not in args: - tester_path = os.path.join(root_dir, os.path.basename(__file__)) - mtime = os.path.getmtime(tester_path) - subprocess.run(["python3", "update_envsetup.py"]) - if mtime != os.path.getmtime(tester_path): - log("The script has changed, restarting it...") - os.execl("/usr/bin/python3", "python3", tester_path, "-n", *args) - sys.exit(1) # not reachable - # Install utilities packages - if "-p" not in args: - subprocess.run(["python3", "pkgs_envsetup.py"]) - # Load conf - conf = utils.load_conf() - if not conf: - log("No configuration loaded.") - sys.exit(1) - # Check for email value - email = "-e" in args - email_if_fail = "-f" in args - basic_only = "-b" in args - tests = self.discover_tests(basic_only, msuser, args) - if not tests: - sys.exit(1) - - if raid_idle(): - exit_code = self.run_tests(tests, email, email_if_fail) - else: - print("A RAID check or operation is in progress, aborting tests") - exit_code = 1 - sys.exit(exit_code) - - def parse_file_header(self, path): - with open(path, "r") as fo: - content = fo.read() - description = "" - if path.endswith(".py"): - start = ( - content.find("'''") - if content.find("'''") != -1 - else content.find('"""') - ) - if start > 0: - start += 3 - end = ( - content.find("'''", start) - if content.find("'''", start) != -1 - else content.find('"""', start) - ) - if end > 0: - description = content[start:end] - else: - for line in content.split("\n"): - if line.startswith("#!"): - continue - elif line.startswith("#"): - description += line[1:].strip() + "\n" - else: - break - description = description.strip() - if description.startswith("Criticality:"): - criticality, *description = description.split("\n") - criticality = criticality[len("Criticality:") :].strip() # noqa: E203 - description = "\n".join(description) - else: - criticality = "not specified" - return criticality, description - - def discover_tests(self, basic_only=False, msuser=None, args=[]): - ignored_tests = utils.get_conf("TESTER_IGNORED_TESTS", "").split(",") - ignored_tests.append("__init__.py") - # Get standard tests - path = os.path.join(self.root_dir, "tests") - if not os.path.isdir(path): - log('The tests dir is missing ("%s").' % path) - return - names = os.listdir(path) - names.sort() - if not names: - log('The tests dir is empty ("%s").' % path) - return - criticalities_map = {"Low": 1, "Normal": 2, "High": 3} - tests = list() - for name in names: - if name in ignored_tests: - continue - test_path = os.path.join(path, name) - if os.path.isfile(test_path): - criticality, description = self.parse_file_header(test_path) - tests.append((name, criticality, description, [test_path])) - if basic_only: - tests.sort(key=lambda i: (-criticalities_map.get(i[1], 0), i[0])) - return tests - elif msuser: - tests = list() - # Get MS instances - ms_users = list() - for user in os.listdir("/home"): - if os.path.exists("/home/%s/msinstance" % user) and ( - not msuser or user == msuser - ): - ms_users.append(user) - # Get MediaServer tests - if ms_users: - ms_users.sort() - cleaned_list = list() - instances_to_test = utils.get_conf("TESTER_MS_INSTANCES", "").split(",") - if instances_to_test: - for val in instances_to_test: - val = val.strip() - if not val: - continue - if val in ms_users: - cleaned_list.append(val) - else: - log( - 'An inexisting instance has been requested for tests: "%s".' - % val - ) - if cleaned_list: - ms_users = cleaned_list - else: - try: - max_instances = int(utils.get_conf("TESTER_MAX_INSTANCES") or 2) - except Exception as e: - log("TESTER_MAX_INSTANCES has an invalid value: %s" % e) - max_instances = 2 - if len(ms_users) > max_instances: - ms_users = ms_users[:max_instances] - log("Instances that will be tested: %s." % ", ".join(ms_users)) - # Clone testing suite - ms_path = os.path.join(path, "ms-testing-suite") - if not os.path.exists(ms_path): - log('Cloning ms-testing-suite in "%s".' % ms_path) - subprocess.run( - [ - "git", - "clone", - "--recursive", - "https://mirismanager.ubicast.eu/git/mediaserver/ms-testing-suite.git", - ms_path, - ] - ) - if os.path.exists(ms_path) and "-n" not in args: - log('Updating ms-testing-suite in "%s".' % ms_path) - os.chdir(ms_path) - branch = utils.get_conf("ENVSETUP_BRANCH") or "stable" - if branch: - subprocess.run(["git", "checkout", branch]) - subprocess.run(["git", "fetch", "--recurse-submodules", "--all"]) - subprocess.run(["git", "reset", "--hard", "origin/{}".format(branch)]) - subprocess.run(["git", "pull", "--recurse-submodules"]) - subprocess.run(["git", "submodule", "update", "--init", "--recursive"]) - os.chdir(self.root_dir) - # Add tests to list - log("Add MediaServer tests if available.") - wowza_dir = "/usr/local/WowzaStreamingEngine" - etc_lives_conf = "/etc/mediaserver/lives_conf.py" - local_lives_conf = "/home/%s/msinstance/conf/lives_conf.py" - for user in ms_users: - ms_tests = ["ms_vod_tester.py", "test_caches.py"] - # Check if live tests should be started - if ( - os.path.exists(wowza_dir) - or os.path.exists(etc_lives_conf) - or os.path.exists(local_lives_conf % user) - ): - ms_tests.append("test_wowza_secure.py") - ms_tests.append("ms_live_tester.py") - for name in ms_tests: - if name in ignored_tests: - continue - test_path = os.path.join(ms_path, name) - criticality, description = self.parse_file_header(test_path) - tests.append( - ( - "%s (%s)" % (name, user), - criticality, - description, - [test_path, user], - ) - ) - tests.sort(key=lambda i: (-criticalities_map.get(i[1], 0), i[0])) - return tests - - def run_tests(self, tests, email=False, email_if_fail=False): # noqa: C901 - - # Run all tests - successes = 0 - failures = 0 - total_duration = None - report_rows = [("Test", "Criticality", "Result", "Duration", "Description")] - report_rows_length = [len(t) for t in report_rows[0]] - out_of_support = False - for name, criticality, description, command in tests: - log('\033[1;95m-- Test "%s" --\033[0;0m' % name) - start_date = datetime.datetime.utcnow() - log("Test start: %s UTC." % start_date.strftime("%Y-%m-%d %H:%M:%S")) - # Run test - count = 0 - while count < 3: - count += 1 - log("Attempt: %s" % str(count)) - p = subprocess.run( - command, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - out = p.stdout.decode("utf-8", "replace").strip() - log(out) - if p.returncode in (0, 2, 3): - break - time.sleep(5 * count * count) - out_of_support = out_of_support or "out of support" in out - if p.returncode == 0: - status = "\033[92msuccess\033[0m" - successes += 1 - elif p.returncode == 2: - status = "\033[94mnot testable\033[0m" - elif p.returncode == 3: - status = "\033[93mwarning\033[0m" - else: - status = "\033[91mfailure\033[0m" - failures += 1 - log("Command exited with code %s." % p.returncode) - # Get duration - end_date = datetime.datetime.utcnow() - duration = end_date - start_date - if total_duration: - total_duration += duration - else: - total_duration = duration - log( - "Test end: %s UTC (duration: %s)." - % (end_date.strftime("%Y-%m-%d %H:%M:%S"), duration) - ) - # Prepare report - report_rows.append((name, criticality, status, str(duration), description)) - report_rows_length = [ - max(len(strip_colors(t)), report_rows_length[i]) - for i, t in enumerate(report_rows[-1]) - ] - # Display results - # results as text - log("\nTests results:") - log_report = "" - for row in report_rows: - if not log_report: - log_report += "-" * 50 - for i, val in enumerate(row): - if i == len(row) - 1: - break - if i == 0: - # merge name and description - log_report += "\n\033[96m%s\033[0m \033[37m%s\033[0m\n" % ( - val, - row[-1], - ) - else: - nb_sp = report_rows_length[i] - len(strip_colors(val)) - log_report += " %s%s" % (val, " " * nb_sp) - log_report += "\n" + "-" * 50 - if out_of_support: - log_report = OUT_OF_SUPPORT_TEXT + "\n" + log_report - log(log_report.strip()) - log("Total tests duration: %s.\n" % total_duration) - # results as html - html_report = "" - for row in report_rows: - html_cell = "th" if not html_report else "td" - html_report += "\n <tr>" - for i, val in enumerate(row): - html_report += " <%s>%s</%s>" % (html_cell, escape(val), html_cell) - html_report += " </tr>" - html_report = '<table border="1">%s\n</table>' % html_report - if out_of_support: - html_report = "<p>" + escape(OUT_OF_SUPPORT_TEXT) + "</p>\n" + html_report - # Store locally results - now = datetime.datetime.utcnow() - log_dir = os.path.join(self.root_dir, "logs") - if not os.path.exists(log_dir): - os.makedirs(log_dir) - history_file = os.path.join(log_dir, "tests_history.txt") - add_header = not os.path.exists(history_file) - with open(history_file, "a") as fo: - if add_header: - fo.write("Date | Result | Succeeded | Failed | Not testable\n") - fo.write( - "%s | %s | %s | %s | %s\n" - % ( - now.strftime("%Y-%m-%d %H:%M:%S"), - "KO" if failures > 0 else "OK", - successes, - failures, - len(tests) - successes - failures, - ) - ) - # Search for old logs to remove - names = os.listdir(log_dir) - names.sort() - for name in list(names): - if not name.startswith("results_"): - names.remove(name) - while len(names) > self.MAX_LOG_FILES - 1: - name = names.pop(0) - try: - log('Removing old log "%s".' % os.path.join(log_dir, name)) - os.remove(os.path.join(log_dir, name)) - except Exception as e: - log("Failed to remove old log: %s" % e) - # Write log to file - hostname = socket.gethostname() - if not hostname: - log("Failed to get hostname (required to send email).") - log_name = "results_%s_%s.txt" % ( - hostname or "noname", - now.strftime("%Y-%m-%d_%H-%M-%S"), - ) - log_content = strip_colors(log_buffer.getvalue()) - log_content_encoding = "utf-8" - with open(os.path.join(log_dir, log_name), "w") as fo: - fo.write(log_content) - # Send email - send_email = False - if hostname: - if email: - send_email = True - elif email_if_fail and failures > 0: - # if they were too many consecutive failures, do not send the email - with open(history_file, "r") as fo: - history_content = fo.read() - lines = history_content.split("\n") - lines.reverse() - consecutive_failures = 0 - for line in lines: - if line: - if "KO" in line: - consecutive_failures += 1 - else: - break - if consecutive_failures == self.NO_MAIL_FAILURES_COUNT: - consecutive_msg = ( - "Maximum consecutive tester failures reached (%s).\nNo more emails will be sent." - % consecutive_failures - ) - send_email = True - elif consecutive_failures < self.NO_MAIL_FAILURES_COUNT: - consecutive_msg = ( - "Consecutive tester failures: %s (will stop sending reports when reaching %s failures)." - % (consecutive_failures, self.NO_MAIL_FAILURES_COUNT) - ) - send_email = True - else: - consecutive_msg = ( - "Too many consecutive tester failures: %s, no email will be sent." - % consecutive_failures - ) - log(consecutive_msg) - html_report += "\n<br/>" + consecutive_msg.replace("\n", "\n<br/>") - if send_email: - sender = "support@ubicast.eu" - recipients = utils.get_conf("EMAIL_ADMINS") or "" - system_domain = utils.get_conf("MS_SERVER_NAME") - system_type = "MediaServer" - if system_domain == "mediaserver": - system_domain = utils.get_conf("CM_SERVER_NAME") - system_type = "MirisManager" - if system_domain == "mirismanager": - system_domain = utils.get_conf("MONITOR_SERVER_NAME") - system_type = "Server" - if system_domain == "monitor": - system_type = "-" - if out_of_support: - recipients = recipients.replace("sysadmin@ubicast.eu", "").replace( - ",,", "," - ) - elif utils.get_conf("PREMIUM_SUPPORT") != "0": - system_domain = "[PREMIUM] %s" % system_domain - recipients = recipients.replace("sysadmin@ubicast.eu", "").replace( - ",,", "," - ) - recipients += ",sysadmin+premium@ubicast.eu" - recipients = recipients.strip(",") - if not recipients: - log( - "No recipients defined for email sending. Set a value for EMAIL_ADMINS." - ) - return 1 - boundary = str(uuid.uuid4()) - if utils.get_conf("TESTER_BASE64_ATTACH") != "0": - log_content_encoding = "base64" - log_content = base64.b64encode(log_content.encode("utf-8")).decode() - mail = """From: %(hostname)s <%(sender)s> -To: %(recipients)s -Subject: %(system_domain)s (%(hostname)s) %(system_type)s health report: %(status)s -Mime-Version: 1.0 -Content-type: multipart/related; boundary="%(boundary)s" - ---%(boundary)s -Content-Type: text/html; charset=UTF-8 -Content-transfer-encoding: utf-8 - -<p><b>Date: %(date)s UTC</b></p> -%(report)s - ---%(boundary)s -Content-type: text/plain; name="%(log_name)s"; charset=UTF-8 -Content-disposition: attachment; filename="%(log_name)s" -Content-transfer-encoding: %(log_content_encoding)s - -%(log_content)s""" % dict( - boundary=boundary, - sender=sender, - hostname=hostname, - recipients=recipients, - status=("KO (%s tests failed)" % failures) if failures > 0 else "OK", - date=now.strftime("%Y-%m-%d %H:%M:%S"), - report=html_report, - log_name=log_name, - log_content_encoding=log_content_encoding, - log_content=log_content, - system_domain=system_domain, - system_type=system_type, - ) - p = subprocess.Popen( - ["/usr/sbin/sendmail", "-t"], - stdin=subprocess.PIPE, - stdout=sys.stdout.stream, - stderr=sys.stderr.stream, - ) - p.communicate(input=mail.encode("utf-8")) - if p.returncode != 0: - log("Failed to send email.") - return 1 - else: - log("Email sent to: %s" % recipients) - exit_code = 1 if failures > 0 else 0 - return exit_code - - -if __name__ == "__main__": - Tester(*sys.argv[1:]) diff --git a/tester.py b/tester.py new file mode 120000 index 00000000..a07060be --- /dev/null +++ b/tester.py @@ -0,0 +1 @@ +tests/tester.py \ No newline at end of file diff --git a/pkgs_envsetup.py b/tests/pkgs_envsetup.py similarity index 100% rename from pkgs_envsetup.py rename to tests/pkgs_envsetup.py diff --git a/__init__.py b/tests/scripts/__init__.py similarity index 100% rename from __init__.py rename to tests/scripts/__init__.py diff --git a/tests/test_apt.py b/tests/scripts/test_apt.py similarity index 73% rename from tests/test_apt.py rename to tests/scripts/test_apt.py index 48d15f44..67cbad5a 100755 --- a/tests/test_apt.py +++ b/tests/scripts/test_apt.py @@ -21,9 +21,9 @@ except ImportError: sys.path.append(str(Path(__file__).parents[1].resolve())) -import utils as u # noqa: E402 -from utils_lib.apt import Apt # noqa: E402 -from utils_lib.os import line_in_file # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.apt import Apt # noqa: E402 +from utilities.os import line_in_file # noqa: E402 def main(): @@ -39,48 +39,48 @@ def main(): apt = Apt(update=True) except apt_mod.cache.FetchFailedException as apt_cache_err: if str(apt_cache_err).endswith("no longer has a Release file."): - u.error("system out of support") + lg.error("system out of support") errors += 1 else: - u.error("Apt error: {}".format(apt_cache_err)) + lg.error("Apt error: {}".format(apt_cache_err)) errors += 1 apt = Apt() # detect pending upgrade upgradable = len(apt.upgradable_packages) if upgradable: - u.info("there is {} upgrade pending".format(upgradable)) + lg.info("there is {} upgrade pending".format(upgradable)) else: - u.success("system up-to-date") + lg.success("system up-to-date") # detect pending auto-remove removable = len(apt.removable_packages) if removable: - u.info("there is {} auto-removable packages".format(removable)) + lg.info("there is {} auto-removable packages".format(removable)) for pkg in apt.removable_packages: if "ubicast" in pkg: - u.error("the ubicast package '%s' can be auto-removed!" % pkg) + lg.error("the ubicast package '%s' can be auto-removed!" % pkg) errors += 1 else: - u.success("system clean") + lg.success("system clean") # detect rc state purgeable = len(apt.purgeable_packages) if purgeable: - u.info("there is {} packages in rc state".format(purgeable)) + lg.info("there is {} packages in rc state".format(purgeable)) # installation try: installed = apt.install("sl") except apt_pkg.Error as apt_install_err: - u.warning(apt_install_err) + lg.warning(apt_install_err) warnings += 1 else: if installed: - u.success("installation successful") + lg.success("installation successful") apt.remove("sl") else: - u.error("installation failed") + lg.error("installation failed") errors += 1 # unattended-upgrades @@ -100,9 +100,9 @@ def main(): "/etc/apt/apt.conf.d/50unattended-upgrades", ) ): - u.success("automatic security updates enabled") + lg.success("automatic security updates enabled") else: - u.warning("automatic security updates not enabled") + lg.warning("automatic security updates not enabled") warnings += 1 # check ubicast repository presence @@ -123,15 +123,15 @@ def main(): else False ) if ubicast_repo and ubicast_package: - u.success("ubicast repository present") + lg.success("ubicast repository present") elif not ubicast_repo and ubicast_package: - u.error("ubicast repository missing") + lg.error("ubicast repository missing") errors += 1 elif not ubicast_repo and not ubicast_package: - u.info("no ubicast repository and service installed") + lg.info("no ubicast repository and service installed") return 2 else: - u.info("no ubicast service installed") + lg.info("no ubicast service installed") # check ubicast repository url regexp_repo = ( @@ -140,19 +140,19 @@ def main(): repo_url_match = line_in_file(regexp_repo, "/etc/apt/sources.list.d/skyreach.list") if repo_url_match: url, apt_token = repo_url_match.groups() - u.success("url: {}, token: {}[...]".format(url, apt_token[:8])) + lg.success("url: {}, token: {}[...]".format(url, apt_token[:8])) else: url, apt_token = None, None - u.error("incorrect ubicast repository url or token") + lg.error("incorrect ubicast repository url or token") errors += 1 # check server avalability if url: server_response = requests.get(url, verify=False) if server_response.ok: - u.success("request to {} succeeded".format(url)) + lg.success("request to {} succeeded".format(url)) else: - u.error("request to {} failed: {}".format(url, server_response.text)) + lg.error("request to {} failed: {}".format(url, server_response.text)) errors += 1 # check repository avalability @@ -161,9 +161,9 @@ def main(): repo_response = requests.get(apt_url, verify=False) apt_url = "{}/packaging/apt/{}[...]/Packages".format(url, apt_token[:8]) if repo_response.ok: - u.success("request to {} succeeded".format(apt_url)) + lg.success("request to {} succeeded".format(apt_url)) else: - u.error("request to {} failed: {}".format(apt_url, repo_response.text)) + lg.error("request to {} failed: {}".format(apt_url, repo_response.text)) errors += 1 if errors: diff --git a/tests/test_apt_proxy.py b/tests/scripts/test_apt_proxy.py similarity index 77% rename from tests/test_apt_proxy.py rename to tests/scripts/test_apt_proxy.py index 9f7d7bc6..81186b78 100755 --- a/tests/test_apt_proxy.py +++ b/tests/scripts/test_apt_proxy.py @@ -19,14 +19,14 @@ except ImportError: sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 def main(): # get Miris Manager domain path = '/etc/nginx/sites-enabled/skyreach.conf' if not os.path.exists(path): - u.log('Server not running Miris Manager, skipping test') + lg.log('Server not running Miris Manager, skipping test') return 2 domain = None with open(path, 'r') as fo: @@ -34,20 +34,20 @@ def main(): if line.strip().startswith('server_name'): domain = line.strip()[len('server_name'):].strip(' \t;').split(' ')[0] if not domain: - u.error('Miris Manager domain not found in Nginx configuration.') + lg.error('Miris Manager domain not found in Nginx configuration.') return 1 try: url = 'https://%s/mirismanager.ubicast.eu/old-releases.ubuntu.com/ubuntu/dists/lucid/Release.gpg' % domain - u.log('Checking url certificate "%s"...' % url) + lg.log('Checking url certificate "%s"...' % url) response = requests.get(url, verify=False).text if 'BEGIN PGP SIGNATURE' not in response: - u.error('Unexpected content:\n%s' % response) + lg.error('Unexpected content:\n%s' % response) return 1 else: - u.success('Test OK.') + lg.success('Test OK.') except Exception as e: - u.error('Package mirror not working: %s' % e) + lg.error('Package mirror not working: %s' % e) return 1 return 0 diff --git a/tests/test_backup.py b/tests/scripts/test_backup.py similarity index 86% rename from tests/test_backup.py rename to tests/scripts/test_backup.py index db86df0e..a621665c 100755 --- a/tests/test_backup.py +++ b/tests/scripts/test_backup.py @@ -16,7 +16,8 @@ import sys sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.config import load_conf # noqa: E402 MAX_AGE = 2 @@ -37,22 +38,22 @@ def test_ssh(host: str) -> bool: ) try: subprocess.check_output(cmd, shell=True, timeout=5) - u.success("logged in successfully") + lg.success("logged in successfully") except subprocess.CalledProcessError: - u.error("failed to login using SSH public key authentication") + lg.error("failed to login using SSH public key authentication") return False except subprocess.TimeoutExpired: - u.error("timeout") + lg.error("timeout") try: cmd_port = "nc -z -w2 {} 22".format(host) subprocess.check_output(cmd_port, shell=True, timeout=5) except subprocess.CalledProcessError: - u.error("failed to bind SSH port") + lg.error("failed to bind SSH port") try: cmd_ping = "ping -c2 -w4 {}".format(host) subprocess.check_output(cmd_ping, shell=True, timeout=15) except subprocess.CalledProcessError: - u.error("failed to ping host") + lg.error("failed to ping host") return False return True @@ -100,13 +101,13 @@ def test_last_backup_is_recent(server: str) -> bool: date = datetime.strptime(last, "%Y-%m-%d-%H%M%S") # check age if (datetime.now() - date).days > MAX_AGE: - u.error("older than {} days: {}".format(MAX_AGE, date)) + lg.error("older than {} days: {}".format(MAX_AGE, date)) return False - u.success("less than {} days old".format(MAX_AGE)) + lg.success("less than {} days old".format(MAX_AGE)) return True # if we reach here, nothing have been found - u.error("latest backup directory not found") + lg.error("latest backup directory not found") return False @@ -127,11 +128,11 @@ def check_backup_is_incremental(path: str) -> bool: if os.path.isdir(folder_path): files_count = len(os.listdir(folder_path)) if files_count == 0: - u.error("folder {} is empty".format(folder_path)) + lg.error("folder {} is empty".format(folder_path)) os.rmdir(folder_path) all_ok = False if all_ok: - u.success("no incrementation issue found") + lg.success("no incrementation issue found") return all_ok @@ -152,7 +153,7 @@ def check_local_backup(path: str) -> bool: latest = os.path.join(backup_folder, "latest") if backup_folder.endswith(".disabled"): # skip if disabled - u.info("disabled") + lg.info("disabled") elif os.path.exists(latest): # resolve symbolic link latest = os.path.realpath(latest) @@ -161,17 +162,17 @@ def check_local_backup(path: str) -> bool: now = datetime.now() diff_seconds = (now - date).total_seconds() if diff_seconds > MAX_AGE * 24 * 3600: - u.error("older than {} days: {}".format(MAX_AGE, date)) + lg.error("older than {} days: {}".format(MAX_AGE, date)) all_ok = False else: - u.success("less than {} days old".format(MAX_AGE)) + lg.success("less than {} days old".format(MAX_AGE)) if not check_backup_is_incremental(backup_folder): all_ok = False elif os.path.exists(os.path.join(backup_folder, "backup.inprogress")): - u.warning("still running") + lg.warning("still running") all_ok = False else: - u.error("not working") + lg.error("not working") all_ok = False return all_ok @@ -200,7 +201,7 @@ def check_local_backups(paths: str) -> bool: def main(): """Run all checks and exits with corresponding exit code.""" - conf = u.load_conf() + conf = load_conf() backup_server = conf.get("BACKUP_SERVER") local_backup_folders = conf.get("LOCAL_BACKUP_FOLDERS") if backup_server: diff --git a/tests/test_dns_records.py b/tests/scripts/test_dns_records.py similarity index 86% rename from tests/test_dns_records.py rename to tests/scripts/test_dns_records.py index aef6be3a..191d6da1 100755 --- a/tests/test_dns_records.py +++ b/tests/scripts/test_dns_records.py @@ -18,8 +18,9 @@ except ImportError: sys.path.append(str(Path(__file__).parents[1].resolve())) -import utils as u # noqa: E402 -from utils_lib.os import supported_platform # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.config import load_conf # noqa: E402 +from utilities.os import supported_platform # noqa: E402 def get_dns_servers() -> set: @@ -87,14 +88,14 @@ def check_dns(hostname: str, expected_ip: str, resolvers: set) -> tuple: try: answers = [rdata.address for rdata in resolver.query(hostname)] except Exception as dns_err: - u.error("cannot resolve {}: {}".format(hostname, dns_err)) + lg.error("cannot resolve {}: {}".format(hostname, dns_err)) errors += 1 else: for address in answers: if address == expected_ip: - u.success("{}".format(address)) + lg.success("{}".format(address)) else: - u.error("{} instead of {}".format(address, expected_ip)) + lg.error("{} instead of {}".format(address, expected_ip)) errors += 1 return warnings, errors @@ -117,13 +118,13 @@ def check_resolver(conf: dict, resolvers: set) -> tuple: if conf_resolver: resolver_set = True if conf_resolver not in resolvers: - u.warning("resolver {} not configured".format(conf_resolver)) + lg.warning("resolver {} not configured".format(conf_resolver)) warnings += 1 else: - u.success("resolver {} configured".format(conf_resolver)) + lg.success("resolver {} configured".format(conf_resolver)) if not resolver_set: - u.info("no resolver defined in envsetup") + lg.info("no resolver defined in envsetup") exit(2) return warnings, errors @@ -133,12 +134,12 @@ def main(): print("Check DNS settings:") if not supported_platform(): - u.info("platform not supported") + lg.info("platform not supported") exit(2) warnings = 0 errors = 0 - conf = u.load_conf() + conf = load_conf() resolvers = get_dns_servers() ip = conf.get("NETWORK_IP_NAT") or conf.get("NETWORK_IP") @@ -155,7 +156,7 @@ def main(): ) if not ip: - u.info("no ip address defined in envsetup") + lg.info("no ip address defined in envsetup") exit(2) for conf_name, default_domain, package in services_info: @@ -169,14 +170,14 @@ def main(): # check that the service is installed on this system status, _ = subprocess.getstatusoutput("dpkg -s {}".format(package)) if status == 0 and ip: - u.info("resolving {}".format(domain)) + lg.info("resolving {}".format(domain)) check_dns_warn, check_dns_err = check_dns(domain, ip, resolvers) if check_dns_err: errors += check_dns_err if check_dns_warn: warnings += check_dns_warn else: - u.info("{} not installed, skip {}".format(package, domain)) + lg.info("{} not installed, skip {}".format(package, domain)) if errors: exit(1) diff --git a/tests/test_email.py b/tests/scripts/test_email.py similarity index 80% rename from tests/test_email.py rename to tests/scripts/test_email.py index 81ecc114..d0891113 100755 --- a/tests/test_email.py +++ b/tests/scripts/test_email.py @@ -18,7 +18,10 @@ import spf sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.commands import exec_cmd # noqa: E402 +from utilities.config import load_conf # noqa: E402 +from utilities.network import get_ip # noqa: E402 def check_listen() -> tuple: @@ -35,10 +38,10 @@ def check_listen() -> tuple: status, out = subprocess.getstatusoutput("ss -pant | grep master | grep ':25'") if status != 0 or ("127.0.0.1:25" not in out and "[::1]:25" not in out): - u.warning("Postfix is not listening on localhost:25") + lg.warning("Postfix is not listening on localhost:25") warnings += 1 else: - u.success("Postfix is listening on localhost:25") + lg.success("Postfix is listening on localhost:25") return warnings, errors @@ -81,21 +84,21 @@ def check_relay(relay_host: str, relay_port: str, domain: str) -> tuple: origins = set( ( domain or None, - u.exec_cmd("hostname", log_output=False)[1] or None, - u.exec_cmd("hostname -f", log_output=False)[1] or None, + exec_cmd("hostname", log_output=False)[1] or None, + exec_cmd("hostname -f", log_output=False)[1] or None, ) ) if myorigin not in origins: - u.warning('"myorigin" setting does not contain a valid domain') + lg.warning('"myorigin" setting does not contain a valid domain') warnings += 1 relay = "{}:{}".format(relay_host, relay_port) if relay_port else relay_host if relay != configured_relay: - u.error("STMP relay must be {}".format(relay)) + lg.error("STMP relay must be {}".format(relay)) errors += 1 if not errors and not warnings: - u.success("STMP relay is properly set") + lg.success("STMP relay is properly set") return warnings, errors @@ -117,7 +120,7 @@ def check_send(sender: str) -> tuple: if sender: sender = "-a 'From: {}' ".format(sender) else: - u.info("Sender address is not set") + lg.info("Sender address is not set") cmd = "echo 'test email' | mail -s 'Email used to test configuration.' {}{}".format( sender, email ) @@ -129,9 +132,9 @@ def check_send(sender: str) -> tuple: elif Path("/var/log/mail.log").is_file(): cmd = "grep '{}' /var/log/mail.log".format(email) else: - u.info("/var/log/mail.log not found, trying journalctl") + lg.info("/var/log/mail.log not found, trying journalctl") cmd = "journalctl -t postfix/smtp | grep {}".format(email) - u.log("Using following command to search for sending log:\n{}".format(cmd)) + lg.log("Using following command to search for sending log:\n{}".format(cmd)) # init vars timeout = 120 @@ -139,7 +142,7 @@ def check_send(sender: str) -> tuple: delay = 1 timed_out = False out = "" - u.log("Email sending timeout is {} seconds.".format(timeout)) + lg.log("Email sending timeout is {} seconds.".format(timeout)) # logs polling sys.stdout.write("Waiting for sending log") @@ -166,21 +169,21 @@ def check_send(sender: str) -> tuple: # check if the sending has timed out if timed_out: - u.error("Failed to send email (timed out).") + lg.error("Failed to send email (timed out).") if out: - u.info("> sending log line:\n{}".format(out)) + lg.info("> sending log line:\n{}".format(out)) else: - u.info("> no log entry found.") + lg.info("> no log entry found.") errors += 1 # check output for errors elif "bounced" in out or "you tried to reach does not exist" in out: - u.error("Failed to send email") - u.info("> sending log line:\n{}".format(out)) + lg.error("Failed to send email") + lg.info("> sending log line:\n{}".format(out)) errors += 1 if not errors: - u.success("Can send email") + lg.success("Can send email") return warnings, errors @@ -202,19 +205,19 @@ def check_spf(ip_addr: str, sender: str, domain: str) -> tuple: errors = 0 if ip_address(ip_addr).is_private: - u.info("{} is a private address, cannot check SPF".format(ip_addr)) + lg.info("{} is a private address, cannot check SPF".format(ip_addr)) elif ip_addr and sender: # check spf result, _ = spf.check2(i=ip_addr, s=domain, h="") if result in ("pass", "neutral"): - u.success("SPF for {} in {}: {}".format(ip_addr, domain, result)) + lg.success("SPF for {} in {}: {}".format(ip_addr, domain, result)) elif result == "none": - u.info("SPF for {} in {}: {}".format(ip_addr, domain, result)) + lg.info("SPF for {} in {}: {}".format(ip_addr, domain, result)) else: - u.warning("SPF for {} in {}: {}".format(ip_addr, domain, result)) + lg.warning("SPF for {} in {}: {}".format(ip_addr, domain, result)) warnings += 1 else: - u.info("IP or sender not set, cannot check SPF") + lg.info("IP or sender not set, cannot check SPF") return warnings, errors @@ -228,11 +231,11 @@ def main(): print("Checking email settings:") if not Path("/etc/postfix").exists(): - u.info("postfix is not installed") + lg.info("postfix is not installed") exit(2) # get settings - conf = u.load_conf() + conf = load_conf() relay = conf.get("EMAIL_SMTP_SERVER", "").replace("[", "").replace("]", "") relay_host = relay.split(":")[0] if ":" in relay else relay relay_port = relay.split(":")[-1] if ":" in relay else "" @@ -240,7 +243,7 @@ def main(): (socket.gethostbyname(relay_host) if relay_host else None) or conf.get("NETWORK_IP_NAT") or conf.get("NETWORK_IP") - or u.get_ip() + or get_ip() or None ) sender = conf.get("EMAIL_SENDER") or "" diff --git a/tests/test_fail2ban.py b/tests/scripts/test_fail2ban.py similarity index 88% rename from tests/test_fail2ban.py rename to tests/scripts/test_fail2ban.py index a50b048a..b5987f0d 100755 --- a/tests/test_fail2ban.py +++ b/tests/scripts/test_fail2ban.py @@ -12,7 +12,8 @@ import sys sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.commands import exec_cmd # noqa: E402 def get_service_state(name: str) -> tuple: @@ -28,7 +29,7 @@ def get_service_state(name: str) -> tuple: # pylint: disable=E0401 import dbus except ImportError: - returncode, output = u.exec_cmd( + returncode, output = exec_cmd( "systemctl status fail2ban | grep 'Active:'", log_output=False ) if returncode != 0: @@ -89,7 +90,7 @@ def get_jails() -> list: :rtype: list """ - _, output = u.exec_cmd( + _, output = exec_cmd( "fail2ban-client status | grep 'Jail list'", log_output=False ) jails = output.split(":")[1].strip().replace(" ", "").split(",") @@ -106,7 +107,7 @@ def check_jail_banned(name: str) -> int: :rtype: int """ - _, output = u.exec_cmd( + _, output = exec_cmd( "fail2ban-client status {} | grep 'Currently banned'".format(name), log_output=False, ) @@ -122,7 +123,7 @@ def main(): """Run all checks and exits with corresponding exit code.""" if subprocess.call(["which", "fail2ban-server"], stdout=subprocess.DEVNULL) != 0: - u.info("fail2ban not installed, skipping test") + lg.info("fail2ban not installed, skipping test") exit(2) # init @@ -131,20 +132,20 @@ def main(): print("Checking fail2ban state:") if not check_service_running("fail2ban"): - u.warning("fail2ban is not running") + lg.warning("fail2ban is not running") warnings += 1 # warning exit if not running exit(3) else: - u.success("fail2ban is running") + lg.success("fail2ban is running") print("Checking fail2ban jails:") jails = get_jails() for jail in jails: - u.info("{} jail is running".format(jail)) + lg.info("{} jail is running".format(jail)) banned = check_jail_banned(jail) if banned > 0: - u.info("there is {} banned host in {} jail".format(banned, jail)) + lg.info("there is {} banned host in {} jail".format(banned, jail)) if errors: exit(1) diff --git a/tests/test_mediaworker.py b/tests/scripts/test_mediaworker.py similarity index 72% rename from tests/test_mediaworker.py rename to tests/scripts/test_mediaworker.py index 7e4640f8..59ac0fd0 100755 --- a/tests/test_mediaworker.py +++ b/tests/scripts/test_mediaworker.py @@ -12,17 +12,17 @@ import sys sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 def check_ssh(ip): cmd = 'ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no root@%s ls /tmp' % ip - u.log('Connecting to MediaWorker:\n%s' % cmd) + lg.log('Connecting to MediaWorker:\n%s' % cmd) try: subprocess.check_output(cmd, shell=True, timeout=5) - u.success('Logged in successfully in "%s".' % ip) + lg.success('Logged in successfully in "%s".' % ip) except subprocess.CalledProcessError: - u.error('Failed to login using SSH, run "ssh-copy-id %s".' % ip) + lg.error('Failed to login using SSH, run "ssh-copy-id %s".' % ip) return False except subprocess.TimeoutExpired: try: @@ -36,9 +36,9 @@ def check_ssh(ip): def check_celerity_connectivity(ip): - u.log('Getting celerity server url.') + lg.log('Getting celerity server url.') cmd = 'ssh -t root@%s cat /etc/celerity/config.py' % ip - u.log(cmd) + lg.log(cmd) try: d = subprocess.check_output(cmd, shell=True, timeout=5, universal_newlines=True) except subprocess.CalledProcessError: @@ -46,53 +46,53 @@ def check_celerity_connectivity(ip): else: m = re.search(r'\nSERVER_URL\s*=\s*([:\/\'\"\-\_\.\w]+)', d) if not m: - u.error('Failed to get celerity tasks server url from configuration in MediaWorker "%s".' % ip) + lg.error('Failed to get celerity tasks server url from configuration in MediaWorker "%s".' % ip) return False server_url = m.groups()[0].strip('"\' ') - u.log('Checking celerity connectivity.') + lg.log('Checking celerity connectivity.') cmd = 'ssh -t root@%s curl -k %s' % (ip, server_url) - u.log(cmd) + lg.log(cmd) try: d = subprocess.check_output(cmd, shell=True, timeout=5, universal_newlines=True) except subprocess.CalledProcessError: d = '' if 'Celerity tasks server' in d: - u.success('Successfully reached tasks server from MediaWorker "%s".' % ip) + lg.success('Successfully reached tasks server from MediaWorker "%s".' % ip) return True - u.error('Failed to reach tasks server from MediaWorker "%s".' % ip) + lg.error('Failed to reach tasks server from MediaWorker "%s".' % ip) return False def check_celerity_versions(ip): - u.log('Checking that celerity server and worker uses the same version.') + lg.log('Checking that celerity server and worker uses the same version.') try: ms_out = subprocess.check_output('dpkg -s celerity-utils | grep "^Version:"', shell=True, timeout=10, universal_newlines=True) mw_out = subprocess.check_output('ssh -t root@%s dpkg -s celerity-utils | grep "^Version:"' % ip, shell=True, timeout=10, universal_newlines=True) except subprocess.CalledProcessError as e: - u.error('Failed to check celerity version in MediaWorker "%s":\n%s' % ip, e) + lg.error('Failed to check celerity version in MediaWorker "%s":\n%s' % ip, e) return False ms_out = (ms_out[len('Version:'):] if ms_out.startswith('Version:') else ms_out).strip() mw_out = (mw_out[len('Version:'):] if mw_out.startswith('Version:') else mw_out).strip() if ms_out != mw_out: - u.error('The celerity version in MediaWorker "%s" is not the same as in MediaServer.\nMediaServer version: \tn%s\nMediaWorker version: \t%s' % (ip, ms_out, mw_out)) + lg.error('The celerity version in MediaWorker "%s" is not the same as in MediaServer.\nMediaServer version: \tn%s\nMediaWorker version: \t%s' % (ip, ms_out, mw_out)) return False - u.success('The celerity version in MediaWorker "%s" is the same as in MediaServer.\nCurrent celerity version is: %s.' % (ip, ms_out)) + lg.success('The celerity version in MediaWorker "%s" is the same as in MediaServer.\nCurrent celerity version is: %s.' % (ip, ms_out)) return True def run_tests(ip): - u.log('Updating envsetup tests on MediaWorker.') + lg.log('Updating envsetup tests on MediaWorker.') cmd = 'ssh -t root@%s /root/envsetup/update_envsetup.py' % ip subprocess.run(cmd, shell=True) - u.log('Running envsetup tests on MediaWorker.') + lg.log('Running envsetup tests on MediaWorker.') cmd = 'ssh -t root@%s /root/envsetup/tester.py' % ip p = subprocess.run(cmd, shell=True) if p.returncode == 0: subprocess.check_output(cmd, shell=True, timeout=60) - u.success('All tests completed on MediaWorker "%s".' % ip) + lg.success('All tests completed on MediaWorker "%s".' % ip) return True else: - u.error('apt-get update failed on MediaWorker "%s".' % ip) + lg.error('apt-get update failed on MediaWorker "%s".' % ip) return False @@ -111,11 +111,11 @@ def get_remote_workers_ips(): if worker['remote_ip']: ips.add(worker['remote_ip']) except Exception as e: - u.error('Failed to get workers list using celerity API: %s' % e) + lg.error('Failed to get workers list using celerity API: %s' % e) # remove local IP p = subprocess.run(['ip', 'addr'], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') local_ips = re.findall(r'inet ([\d\.]+)/', p.stdout) - u.log('Local IP addresses are: %s.' % local_ips) + lg.log('Local IP addresses are: %s.' % local_ips) ips.difference(set(local_ips)) return ips @@ -124,10 +124,10 @@ def main(): try: import mediaserver except ImportError: - u.log('MediaServer is not installed, skipping test.') + lg.log('MediaServer is not installed, skipping test.') return 2 else: - u.log('MediaServer version: %s.' % mediaserver.__version__) + lg.log('MediaServer version: %s.' % mediaserver.__version__) all_ok = True tested = False @@ -146,7 +146,7 @@ def main(): # if not run_tests(worker_ip): # all_ok = False if not tested: - u.log('No remote worker found, skipping test.') + lg.log('No remote worker found, skipping test.') return 2 if not all_ok: diff --git a/tests/test_monitoring.py b/tests/scripts/test_monitoring.py similarity index 74% rename from tests/test_monitoring.py rename to tests/scripts/test_monitoring.py index 9ed63133..68278e0f 100755 --- a/tests/test_monitoring.py +++ b/tests/scripts/test_monitoring.py @@ -13,21 +13,21 @@ import sys sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 MUNIN_WWW_PATH = '/var/cache/munin/www/' def check_munin(): - u.log('Checking if monitoring works...') + lg.log('Checking if monitoring works...') if not os.path.exists(MUNIN_WWW_PATH): p = subprocess.run(['dpkg', '-s', 'munin'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if p.returncode == 0: - u.error('Munin directory "%s" not found.' % MUNIN_WWW_PATH) + lg.error('Munin directory "%s" not found.' % MUNIN_WWW_PATH) return 1 else: - u.info('Munin is not installed, test skipped.') + lg.info('Munin is not installed, test skipped.') return 2 # get cpu day graph of each host @@ -39,22 +39,22 @@ def check_munin(): if sub_name != 'index.html' and os.path.exists(path): paths.append(path) if not paths: - u.error('No Munin host directory was found in "%s".' % MUNIN_WWW_PATH) + lg.error('No Munin host directory was found in "%s".' % MUNIN_WWW_PATH) return 1 # check graph mtime error = False for path in paths: - u.log('Checking graph "%s" modification date...' % path) + lg.log('Checking graph "%s" modification date...' % path) mtime = os.path.getmtime(path) d = datetime.fromtimestamp(mtime) now = datetime.now() diff_seconds = (now - d).total_seconds() if diff_seconds > 3600: - u.error('The graph is older than 1 hour. The monitoring is probably not working.') + lg.error('The graph is older than 1 hour. The monitoring is probably not working.') error = True else: - u.success('The graph is not older than 1 hour.') + lg.success('The graph is not older than 1 hour.') return 1 if error else 0 diff --git a/tests/test_nginx_conf_valid.sh b/tests/scripts/test_nginx_conf_valid.sh similarity index 100% rename from tests/test_nginx_conf_valid.sh rename to tests/scripts/test_nginx_conf_valid.sh diff --git a/tests/test_nginx_status.py b/tests/scripts/test_nginx_status.py similarity index 81% rename from tests/test_nginx_status.py rename to tests/scripts/test_nginx_status.py index aa227ade..75ef254c 100755 --- a/tests/test_nginx_status.py +++ b/tests/scripts/test_nginx_status.py @@ -12,14 +12,14 @@ import sys sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 def main(): print("Checking nginx status:") if not Path("/etc/nginx").exists(): - u.info("nginx dir does not exists, skip test") + lg.info("nginx dir does not exists, skip test") exit(2) try: @@ -33,10 +33,10 @@ def main(): if "Active connections" not in req.text: raise Exception("invalid response from nginx status url") except Exception as e: - u.error(str(e)) + lg.error(str(e)) exit(1) - u.success("status code: {}".format(req.status_code)) + lg.success("status code: {}".format(req.status_code)) exit(0) diff --git a/tests/test_nginx_vhosts.py b/tests/scripts/test_nginx_vhosts.py similarity index 87% rename from tests/test_nginx_vhosts.py rename to tests/scripts/test_nginx_vhosts.py index 8d589215..e8dcac08 100755 --- a/tests/test_nginx_vhosts.py +++ b/tests/scripts/test_nginx_vhosts.py @@ -24,7 +24,8 @@ except ImportError: sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.config import load_conf # noqa: E402 ''' This script checks for all enabled vhosts in Nginx conf that: @@ -94,7 +95,7 @@ def test_vhost( for port, proto in ports_info or [(80, False)]: for domain in domains or ['localhost']: url = '%s://%s:%s' % (proto, domain, port) - u.info('- testing url "%s" from %s' % (url, name)) + lg.info('- testing url "%s" from %s' % (url, name)) # test domain IP ip_error = None ip_warning = None @@ -107,16 +108,16 @@ def test_vhost( ip_warning = '%s resolve to %s instead of 127.0.0.1' % (domain, ip) if ip_error: if resolution_ignored and domain in resolution_ignored: - u.info('%s (ignored)' % ip_error) + lg.info('%s (ignored)' % ip_error) ip_error = None else: - u.error(ip_error) + lg.error(ip_error) elif ip_warning: if resolution_ignored and domain in resolution_ignored: - u.info('%s (ignored)' % ip_warning) + lg.info('%s (ignored)' % ip_warning) ip_warning = None else: - u.warning(ip_warning) + lg.warning(ip_warning) # test url req_error = False try: @@ -135,11 +136,11 @@ def test_vhost( or domain == 'localhost' and code not in (200, 401, 403, 404) ): - u.error('%s status: %s, %s ms' % (domain, code, req_time)) + lg.error('%s status: %s, %s ms' % (domain, code, req_time)) req_error = True else: if req_time > 10000: - u.warning('%s status: %s, %s ms' % (domain, code, req_time)) + lg.warning('%s status: %s, %s ms' % (domain, code, req_time)) warnings += 1 if 'mediaserver' in name and wowza_dir: # test /streaming url @@ -157,10 +158,10 @@ def test_vhost( else: code = req.status_code if code != 200: - u.error('%s streaming: %s, %s ms' % (domain, code, req_time)) + lg.error('%s streaming: %s, %s ms' % (domain, code, req_time)) req_error = True elif req_time > 10000: - u.warning('%s streaming: %s, %s ms' % (domain, code, req_time)) + lg.warning('%s streaming: %s, %s ms' % (domain, code, req_time)) warnings += 1 tested += 1 @@ -178,7 +179,7 @@ def main(): # check that Nginx dir exists nginx_dir = '/etc/nginx/sites-enabled' if not Path(nginx_dir).exists(): - u.info('nginx dir does not exists ("%s"), test skipped.' % nginx_dir) + lg.info('nginx dir does not exists ("%s"), test skipped.' % nginx_dir) exit(2) # check that Wowza is installed @@ -187,7 +188,7 @@ def main(): wowza_dir = None # get envsetup conf - conf = u.load_conf() + conf = load_conf() # get enabled vhosts resolution_ignored = conf.get('TESTER_VHOST_RESOLUTION_IGNORED', '').split(',') @@ -217,7 +218,7 @@ def main(): elif warnings: exit(3) if not tested: - u.error('no url found in nginx sites-enabled dir') + lg.error('no url found in nginx sites-enabled dir') exit(1) diff --git a/tests/test_ntp.py b/tests/scripts/test_ntp.py similarity index 73% rename from tests/test_ntp.py rename to tests/scripts/test_ntp.py index 9d453956..8c977cd1 100755 --- a/tests/test_ntp.py +++ b/tests/scripts/test_ntp.py @@ -14,7 +14,8 @@ import sys sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.config import load_conf # noqa: E402 def main(): @@ -30,15 +31,15 @@ def main(): ntpconf = '/etc/systemd/timesyncd.conf' ntpconf_expected = r'^NTP=(.*)$' - u.log('Running %s' % cmd) + lg.log('Running %s' % cmd) status = subprocess.getoutput(cmd) if expected not in status: - u.error('NTP not working: %s' % status) + lg.error('NTP not working: %s' % status) return 1 - u.success('System is NTP synchronized.') + lg.success('System is NTP synchronized.') - u.log('Checking NTP server conforms to conf...') - conf = u.load_conf() + lg.log('Checking NTP server conforms to conf...') + conf = load_conf() expected_servers = None if conf.get('NTP_SERVER'): expected_servers = [s.strip() for s in conf['NTP_SERVER'].split(',')] @@ -56,11 +57,11 @@ def main(): servers.append(m.groups()[0].strip()) for expected_server in expected_servers: if expected_server not in servers: - u.warning('Warning: Expected NTP server %s not found in %s, found %s instead.' % (expected_server, ntpconf, ', '.join(servers))) + lg.warning('Warning: Expected NTP server %s not found in %s, found %s instead.' % (expected_server, ntpconf, ', '.join(servers))) return 3 else: - u.log('Expected NTP server %s found in configuration (total servers: %s).' % (expected_server, len(servers))) - u.success('NTP OK.') + lg.log('Expected NTP server %s found in configuration (total servers: %s).' % (expected_server, len(servers))) + lg.success('NTP OK.') return 0 diff --git a/tests/test_partitions.py b/tests/scripts/test_partitions.py similarity index 100% rename from tests/test_partitions.py rename to tests/scripts/test_partitions.py diff --git a/tests/test_postgresql.py b/tests/scripts/test_postgresql.py similarity index 88% rename from tests/test_postgresql.py rename to tests/scripts/test_postgresql.py index 2b4cb115..e7e233ba 100755 --- a/tests/test_postgresql.py +++ b/tests/scripts/test_postgresql.py @@ -20,8 +20,9 @@ except ImportError: sys.path.append(str(Path(__file__).parents[1].resolve())) -import utils as u # noqa: E402 -from utils_lib.apt import Apt # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.apt import Apt # noqa: E402 +from utilities.config import load_conf # noqa: E402 def check_listen(host: str, port: int) -> bool: @@ -280,15 +281,15 @@ def check_ha(db_conn: dict, errors: int = 0) -> int: # check haproxy print("Checking local HAProxy frontends:") if not check_listen(db_conn["host"], 54321): - u.error("HAProxy pgsql-primary frontend is not listening") + lg.error("HAProxy pgsql-primary frontend is not listening") errors += 1 else: - u.success("HAProxy pgsql-primary frontend is listening") + lg.success("HAProxy pgsql-primary frontend is listening") if not check_listen(db_conn["host"], 54322): - u.error("HAProxy pgsql-standby frontend is not listening") + lg.error("HAProxy pgsql-standby frontend is not listening") errors += 1 else: - u.success("HAProxy pgsql-standby frontend is listening") + lg.success("HAProxy pgsql-standby frontend is listening") # check remotes print("Checking remote PostgreSQL nodes:") @@ -296,19 +297,19 @@ def check_ha(db_conn: dict, errors: int = 0) -> int: node_host = nodes[node]["host"] node_port = nodes[node]["port"] if not check_listen(node_host, node_port): - u.error("cannot bind {}:{}".format(node_host, node_port)) + lg.error("cannot bind {}:{}".format(node_host, node_port)) errors += 1 else: - u.success("can bind {}:{}".format(node_host, node_port)) + lg.success("can bind {}:{}".format(node_host, node_port)) # check fenced print("Checking cluster state:") fenced, node = check_fenced(nodes) if fenced: - u.error("Node `{}` is fenced".format(node)) + lg.error("Node `{}` is fenced".format(node)) errors += 1 else: - u.success("No fenced node found") + lg.success("No fenced node found") # check replication print("Checking replication state:") @@ -318,10 +319,10 @@ def check_ha(db_conn: dict, errors: int = 0) -> int: standby["port"] = 54322 status, info = check_replication(primary, standby) if not status: - u.error("cannot replicate between primary/standby ({})".format(info)) + lg.error("cannot replicate between primary/standby ({})".format(info)) errors += 1 else: - u.success("can replicate between primary/standby ({})".format(info)) + lg.success("can replicate between primary/standby ({})".format(info)) return errors @@ -344,20 +345,20 @@ def check_local(db_conn: dict, errors: int = 0) -> int: # check listen print("Checking local PostgreSQL node:") if not check_listen(host, port): - u.error("cannot connect to {}:{}".format(host, port)) + lg.error("cannot connect to {}:{}".format(host, port)) errors += 1 else: - u.success("can connect to {}:{}".format(host, port)) + lg.success("can connect to {}:{}".format(host, port)) # check read print("Checking read operation:") read_query = "SELECT 1;" status, info = check_psql(db_conn, read_query) if not status: - u.error("cannot read from {}@{}:{} ({})".format(user, host, port, info)) + lg.error("cannot read from {}@{}:{} ({})".format(user, host, port, info)) errors += 1 else: - u.success("can read from {}@{}:{}".format(user, host, port)) + lg.success("can read from {}@{}:{}".format(user, host, port)) # get replication state if available if check_listen("127.0.0.1", 8543): @@ -368,16 +369,16 @@ def check_local(db_conn: dict, errors: int = 0) -> int: # check write print("Checking write operation:") if state != "primary": - u.info("this database is in {} state".format(state)) + lg.info("this database is in {} state".format(state)) else: rand = uuid.uuid4().hex write_query = "CREATE TABLE es_test_{} (id serial PRIMARY KEY);".format(rand) status, info = check_psql(db_conn, write_query) if not status: - u.error("cannot write on {}@{}:{} ({})".format(user, host, port, info)) + lg.error("cannot write on {}@{}:{} ({})".format(user, host, port, info)) errors += 1 else: - u.success("can write on {}@{}:{}".format(user, host, port)) + lg.success("can write on {}@{}:{}".format(user, host, port)) # remove test table check_psql(db_conn, "DROP TABLE es_test_{};".format(rand)) @@ -392,7 +393,7 @@ def main(): exit(2) # load configuration - conf = u.load_conf() + conf = load_conf() # get database configuration db_host = conf.get("DB_HOST") if conf.get("DB_HOST") else "127.0.0.1" @@ -406,10 +407,10 @@ def main(): # determine if HA setup and run according tests print("Checking availibility mode:") if is_ha(): - u.info("this setup is using a HA database") + lg.info("this setup is using a HA database") errors = check_ha(db_conn) else: - u.info("this setup is using a local database") + lg.info("this setup is using a local database") errors = check_local(db_conn) if errors: diff --git a/tests/test_raid.py b/tests/scripts/test_raid.py similarity index 100% rename from tests/test_raid.py rename to tests/scripts/test_raid.py diff --git a/tests/test_ssl.py b/tests/scripts/test_ssl.py similarity index 82% rename from tests/test_ssl.py rename to tests/scripts/test_ssl.py index 2baf8557..f917fbfe 100755 --- a/tests/test_ssl.py +++ b/tests/scripts/test_ssl.py @@ -17,17 +17,18 @@ import OpenSSL sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 +from utilities.config import load_conf # noqa: E402 def main(): print("Check TLS settings:") if subprocess.call(["which", "nginx"], stdout=subprocess.DEVNULL) != 0: - u.info("nginx not found, skipping test") + lg.info("nginx not found, skipping test") exit(2) - conf = u.load_conf() + conf = load_conf() conf_servers = ( ("MS_SERVER_NAME", "mediaserver"), @@ -70,29 +71,29 @@ def main(): remaining = expires - datetime.datetime.utcnow() if remaining < datetime.timedelta(days=0): - u.error("{}: expired since {}".format(server_name, str(remaining))) + lg.error("{}: expired since {}".format(server_name, str(remaining))) # if mediaserver (the only cert that is mandatory) if setting == conf_servers[0]: failure = True elif remaining < datetime.timedelta(days=7): - u.error("{}: expire in {}".format(server_name, str(remaining))) + lg.error("{}: expire in {}".format(server_name, str(remaining))) # if mediaserver (the only cert that is mandatory) if setting == conf_servers[0]: failure = True elif remaining < datetime.timedelta(days=30): - u.warning("{}: expire in {}".format(server_name, str(remaining))) + lg.warning("{}: expire in {}".format(server_name, str(remaining))) # if mediaserver (the only cert that is mandatory) if setting == conf_servers[0]: all_ok = False else: - u.success("{}: expire in {}".format(server_name, str(remaining))) + lg.success("{}: expire in {}".format(server_name, str(remaining))) try: url = "https://{}".format(name) requests.get(url) - u.success("{}: trusted certificate".format(name)) + lg.success("{}: trusted certificate".format(name)) except requests.exceptions.SSLError: - u.warning("{}: untrusted certificate".format(name)) + lg.warning("{}: untrusted certificate".format(name)) # if mediaserver (the only cert that is mandatory) if setting == conf_servers[0]: all_ok = False diff --git a/tests/test_wowza.py b/tests/scripts/test_wowza.py similarity index 84% rename from tests/test_wowza.py rename to tests/scripts/test_wowza.py index 9c44663a..59071fce 100755 --- a/tests/test_wowza.py +++ b/tests/scripts/test_wowza.py @@ -16,7 +16,7 @@ from psutil import net_connections sys.path.append(str(Path(__file__).parents[1].resolve())) # pylint: disable=wrong-import-position -import utils as u # noqa: E402 +from utilities import logging as lg # noqa: E402 LATEST_VERSION = "4.7.7" @@ -80,7 +80,7 @@ def check_installed() -> bool: state = out.split()[-1] if state != "install": - u.info("not installed, skip test") + lg.info("not installed, skip test") return False return True @@ -103,19 +103,19 @@ def check_version() -> tuple: for line in out.split("\n"): if line.split()[-1] == "install": if version: - u.error("many Wowza versions installed, keep only latest") + lg.error("many Wowza versions installed, keep only latest") errors += 1 version = ".".join(re.findall(r"\d", line)) if not version: - u.error("cannot find wWowza version") + lg.error("cannot find wWowza version") errors += 1 if version != LATEST_VERSION: - u.warning("using outdated version: {}".format(version)) + lg.warning("using outdated version: {}".format(version)) warnings += 1 else: - u.success("using recommended version: {}".format(LATEST_VERSION)) + lg.success("using recommended version: {}".format(LATEST_VERSION)) return warnings, errors @@ -133,10 +133,10 @@ def check_heap_size() -> tuple: cmd = "grep '<HeapSize>2000M</HeapSize>' /usr/local/WowzaStreamingEngine/conf/Tune.xml" check_heap, _ = subprocess.getstatusoutput(cmd) if check_heap != 0: - u.warning("not using recommended heap size") + lg.warning("not using recommended heap size") warnings += 1 else: - u.success("using recommended heap size") + lg.success("using recommended heap size") return warnings, errors @@ -154,10 +154,10 @@ def check_running() -> tuple: cmd = "systemctl status WowzaStreamingEngine" out = subprocess.getoutput(cmd) if "Active: active (running)" not in out: - u.error("service not running") + lg.error("service not running") errors += 1 else: - u.success("service running") + lg.success("service running") return warnings, errors @@ -183,10 +183,10 @@ def check_listening() -> tuple: # check that system is listening on this port if int(port) not in listening: - u.error("not listening on port {}".format(port)) + lg.error("not listening on port {}".format(port)) errors += 1 else: - u.success("listening on port {}".format(port)) + lg.success("listening on port {}".format(port)) return warnings, errors diff --git a/tests/tester.py b/tests/tester.py new file mode 100755 index 00000000..d0995497 --- /dev/null +++ b/tests/tester.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 + +""" +Script to start tests and to manage their results +""" + +from io import StringIO +import argparse +import base64 +import datetime +import glob +import os +import re +import socket +import subprocess +import sys +import time +import uuid + +from utilities.config import load_conf, get_conf +from utilities.os import get_dir + +OUT_OF_SUPPORT_TEXT = """\033[93mWarning: +The system is out of support, UbiCast will not be notified if errors are detected. +Please contact UbiCast sales team (sales@ubicast.eu) to renew the support contract.\033[0m""" + + +class Logger(object): + def __init__(self, stream, log_buffer): + self.stream = stream + self.log_buffer = log_buffer + + def write(self, text): + self.stream.write(text) + self.stream.flush() + self.log_buffer.write(text) + self.log_buffer.flush() + + def flush(self): + pass + + +log_buffer = StringIO() +sys.stdout = Logger(sys.stdout, log_buffer) +sys.stderr = sys.stdout + + +def strip_colors(text): + return re.sub(r"\033\[[\d;]+m", "", text) + + +def escape(text): + html = text.strip() + html = html.replace("<", "<") + html = html.replace(">", ">") + html = html.replace("\033[90m", '<span style="color: gray;">') + html = html.replace("\033[91m", '<span style="color: red;">') + html = html.replace("\033[92m", '<span style="color: green;">') + html = html.replace("\033[93m", '<span style="color: orange;">') + html = html.replace("\033[94m", '<span style="color: blue;">') + html = html.replace("\033[95m", '<span style="color: purple;">') + html = strip_colors(html) + return html + + +def raid_idle(): + idle = True + devs = glob.glob("/sys/block/md*/md/sync_action") + for d in devs: + with open(d, "r") as f: + sync_state = f.read().strip() + if sync_state != "idle": + idle = False + print("State in %s is %s" % (d, sync_state)) + return idle + + +class Tester(): + MAX_LOG_FILES = 50 + NO_MAIL_FAILURES_COUNT = 5 + + def __init__(self): + print("\033[96m-------------------------------\033[0m") + print("\033[96m- UbiCast applications tester -\033[0m") + print("\033[96m-------------------------------\033[0m") + # parse args + parser = argparse.ArgumentParser(description=__doc__.strip()) + parser.add_argument('-d', '--debug', dest='debug', action='store_true', help='Debug mode (can be started with non root users).') + parser.add_argument('-e', '--email', dest='send_email', action='store_true', help='Send tests report by email.') + parser.add_argument('-f', '--email-fail', dest='send_email_if_fail', action='store_true', help='Send tests report by email if at least one test fails.') + parser.add_argument('-b', '--basic', dest='basic_tests', action='store_true', help='Run only basic tests (exclude mediaserver tests).') + parser.add_argument('-n', '--no-update', dest='no_update', action='store_true', help='Do not update envsetup repository.') + parser.add_argument('-p', '--no-packages', dest='no_packages', action='store_true', help='Do not install packages.') + parser.add_argument('msuser', nargs='?', help='The unix user of the MediaServer instance to test. Default is user specified in configuration or all users if not set.') + args = parser.parse_args() + # Check current dir + root_dir = get_dir(__file__) + if root_dir != "": + os.chdir(root_dir) + self.root_dir = root_dir + # Add to python path + if root_dir not in sys.path: + sys.path.append(root_dir) + # Check that this script is run by root + if os.getuid() != 0 and not args.debug: + print("This script should be run as root user.") + sys.exit(1) + # Update envsetup files + if not args.no_update: + tester_path = os.path.join(root_dir, os.path.basename(__file__)) + mtime = os.path.getmtime(tester_path) + subprocess.run(["python3", "update_envsetup.py"]) + if mtime != os.path.getmtime(tester_path): + print("The script has changed, restarting it...") + os.execl("/usr/bin/python3", "python3", tester_path, "-n", *sys.argv[1:]) + sys.exit(1) # not reachable + # Install utilities packages + if not args.no_packages: + subprocess.run(["python3", "pkgs_envsetup.py"]) + # Load conf + conf = load_conf() + if not conf: + print("No configuration loaded.") + sys.exit(1) + # Check for email value + tests = self.discover_tests(args.basic_tests, msuser=args.msuser, no_update=args.no_update) + if not tests: + sys.exit(1) + + if raid_idle(): + exit_code = self.run_tests(tests, send_email=args.send_email, send_email_if_fail=args.send_email_if_fail) + else: + print("A RAID check or operation is in progress, aborting tests") + exit_code = 1 + sys.exit(exit_code) + + def parse_file_header(self, path): + with open(path, "r") as fo: + content = fo.read() + description = "" + if path.endswith(".py"): + start = ( + content.find("'''") + if content.find("'''") != -1 + else content.find('"""') + ) + if start > 0: + start += 3 + end = ( + content.find("'''", start) + if content.find("'''", start) != -1 + else content.find('"""', start) + ) + if end > 0: + description = content[start:end] + else: + for line in content.split("\n"): + if line.startswith("#!"): + continue + elif line.startswith("#"): + description += line[1:].strip() + "\n" + else: + break + description = description.strip() + if description.startswith("Criticality:"): + criticality, *description = description.split("\n") + criticality = criticality[len("Criticality:") :].strip() # noqa: E203 + description = "\n".join(description) + else: + criticality = "not specified" + return criticality, description + + def discover_tests(self, basic_only=False, msuser=None, no_update=False): + ignored_tests = get_conf("TESTER_IGNORED_TESTS", "").split(",") + ignored_tests.append("__init__.py") + # Get standard tests + path = os.path.join(self.root_dir, "scripts") + if not os.path.isdir(path): + print('The tests dir is missing ("%s").' % path) + return + names = os.listdir(path) + names.sort() + if not names: + print('The tests dir is empty ("%s").' % path) + return + criticalities_map = {"Low": 1, "Normal": 2, "High": 3} + tests = list() + for name in names: + if name in ignored_tests: + continue + test_path = os.path.join(path, name) + if os.path.isfile(test_path): + criticality, description = self.parse_file_header(test_path) + tests.append((name, criticality, description, [test_path])) + if basic_only: + tests.sort(key=lambda i: (-criticalities_map.get(i[1], 0), i[0])) + return tests + elif msuser: + tests = list() + # Get MS instances + ms_users = list() + for user in os.listdir("/home"): + if os.path.exists("/home/%s/msinstance" % user) and ( + not msuser or user == msuser + ): + ms_users.append(user) + # Get MediaServer tests + if ms_users: + ms_users.sort() + cleaned_list = list() + instances_to_test = get_conf("TESTER_MS_INSTANCES", "").split(",") + if instances_to_test: + for val in instances_to_test: + val = val.strip() + if not val: + continue + if val in ms_users: + cleaned_list.append(val) + else: + print( + 'An inexisting instance has been requested for tests: "%s".' + % val + ) + if cleaned_list: + ms_users = cleaned_list + else: + try: + max_instances = int(get_conf("TESTER_MAX_INSTANCES") or 2) + except Exception as e: + print("TESTER_MAX_INSTANCES has an invalid value: %s" % e) + max_instances = 2 + if len(ms_users) > max_instances: + ms_users = ms_users[:max_instances] + print("Instances that will be tested: %s." % ", ".join(ms_users)) + # Clone testing suite + ms_path = os.path.join(path, "ms-testing-suite") + if not os.path.exists(ms_path): + print('Cloning ms-testing-suite in "%s".' % ms_path) + subprocess.run( + [ + "git", + "clone", + "--recursive", + "https://mirismanager.ubicast.eu/git/mediaserver/ms-testing-suite.git", + ms_path, + ] + ) + if os.path.exists(ms_path) and not no_update: + print('Updating ms-testing-suite in "%s".' % ms_path) + os.chdir(ms_path) + branch = get_conf("ENVSETUP_BRANCH") or "stable" + if branch: + subprocess.run(["git", "checkout", branch]) + subprocess.run(["git", "fetch", "--recurse-submodules", "--all"]) + subprocess.run(["git", "reset", "--hard", "origin/{}".format(branch)]) + subprocess.run(["git", "pull", "--recurse-submodules"]) + subprocess.run(["git", "submodule", "update", "--init", "--recursive"]) + os.chdir(self.root_dir) + # Add tests to list + print("Add MediaServer tests if available.") + wowza_dir = "/usr/local/WowzaStreamingEngine" + etc_lives_conf = "/etc/mediaserver/lives_conf.py" + local_lives_conf = "/home/%s/msinstance/conf/lives_conf.py" + for user in ms_users: + ms_tests = ["ms_vod_tester.py", "test_caches.py"] + # Check if live tests should be started + if ( + os.path.exists(wowza_dir) + or os.path.exists(etc_lives_conf) + or os.path.exists(local_lives_conf % user) + ): + ms_tests.append("test_wowza_secure.py") + ms_tests.append("ms_live_tester.py") + for name in ms_tests: + if name in ignored_tests: + continue + test_path = os.path.join(ms_path, name) + criticality, description = self.parse_file_header(test_path) + tests.append( + ( + "%s (%s)" % (name, user), + criticality, + description, + [test_path, user], + ) + ) + tests.sort(key=lambda i: (-criticalities_map.get(i[1], 0), i[0])) + return tests + + def run_tests(self, tests, send_email=False, send_email_if_fail=False): # noqa: C901 + # Run all tests + successes = 0 + failures = 0 + total_duration = None + report_rows = [("Test", "Criticality", "Result", "Duration", "Description")] + report_rows_length = [len(t) for t in report_rows[0]] + out_of_support = False + for name, criticality, description, command in tests: + print('\033[1;95m-- Test "%s" --\033[0;0m' % name) + start_date = datetime.datetime.utcnow() + print("Test start: %s UTC." % start_date.strftime("%Y-%m-%d %H:%M:%S")) + # Run test + count = 0 + while count < 3: + count += 1 + print("Attempt: %s" % str(count)) + p = subprocess.run( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + out = p.stdout.decode("utf-8", "replace").strip() + print(out) + if p.returncode in (0, 2, 3): + break + time.sleep(5 * count * count) + out_of_support = out_of_support or "out of support" in out + if p.returncode == 0: + status = "\033[92msuccess\033[0m" + successes += 1 + elif p.returncode == 2: + status = "\033[94mnot testable\033[0m" + elif p.returncode == 3: + status = "\033[93mwarning\033[0m" + else: + status = "\033[91mfailure\033[0m" + failures += 1 + print("Command exited with code %s." % p.returncode) + # Get duration + end_date = datetime.datetime.utcnow() + duration = end_date - start_date + if total_duration: + total_duration += duration + else: + total_duration = duration + print( + "Test end: %s UTC (duration: %s)." + % (end_date.strftime("%Y-%m-%d %H:%M:%S"), duration) + ) + # Prepare report + report_rows.append((name, criticality, status, str(duration), description)) + report_rows_length = [ + max(len(strip_colors(t)), report_rows_length[i]) + for i, t in enumerate(report_rows[-1]) + ] + # Display results + # results as text + print("\nTests results:") + log_report = "" + for row in report_rows: + if not log_report: + log_report += "-" * 50 + for i, val in enumerate(row): + if i == len(row) - 1: + break + if i == 0: + # merge name and description + log_report += "\n\033[96m%s\033[0m \033[37m%s\033[0m\n" % ( + val, + row[-1], + ) + else: + nb_sp = report_rows_length[i] - len(strip_colors(val)) + log_report += " %s%s" % (val, " " * nb_sp) + log_report += "\n" + "-" * 50 + if out_of_support: + log_report = OUT_OF_SUPPORT_TEXT + "\n" + log_report + print(log_report.strip()) + print("Total tests duration: %s.\n" % total_duration) + # results as html + html_report = "" + for row in report_rows: + html_cell = "th" if not html_report else "td" + html_report += "\n <tr>" + for i, val in enumerate(row): + html_report += " <%s>%s</%s>" % (html_cell, escape(val), html_cell) + html_report += " </tr>" + html_report = '<table border="1">%s\n</table>' % html_report + if out_of_support: + html_report = "<p>" + escape(OUT_OF_SUPPORT_TEXT) + "</p>\n" + html_report + # Store locally results + now = datetime.datetime.utcnow() + log_dir = os.path.join(self.root_dir, "logs") + if not os.path.exists(log_dir): + os.makedirs(log_dir) + history_file = os.path.join(log_dir, "tests_history.txt") + add_header = not os.path.exists(history_file) + with open(history_file, "a") as fo: + if add_header: + fo.write("Date | Result | Succeeded | Failed | Not testable\n") + fo.write( + "%s | %s | %s | %s | %s\n" + % ( + now.strftime("%Y-%m-%d %H:%M:%S"), + "KO" if failures > 0 else "OK", + successes, + failures, + len(tests) - successes - failures, + ) + ) + # Search for old logs to remove + names = os.listdir(log_dir) + names.sort() + for name in list(names): + if not name.startswith("results_"): + names.remove(name) + while len(names) > self.MAX_LOG_FILES - 1: + name = names.pop(0) + try: + print('Removing old log "%s".' % os.path.join(log_dir, name)) + os.remove(os.path.join(log_dir, name)) + except Exception as e: + print("Failed to remove old log: %s" % e) + # Write log to file + hostname = socket.gethostname() + if not hostname: + print("Failed to get hostname (required to send email).") + log_name = "results_%s_%s.txt" % ( + hostname or "noname", + now.strftime("%Y-%m-%d_%H-%M-%S"), + ) + log_content = strip_colors(log_buffer.getvalue()) + log_content_encoding = "utf-8" + with open(os.path.join(log_dir, log_name), "w") as fo: + fo.write(log_content) + # Send email + should_send_email = False + if hostname: + if send_email: + should_send_email = True + elif send_email_if_fail and failures > 0: + # if they were too many consecutive failures, do not send the email + with open(history_file, "r") as fo: + history_content = fo.read() + lines = history_content.split("\n") + lines.reverse() + consecutive_failures = 0 + for line in lines: + if line: + if "KO" in line: + consecutive_failures += 1 + else: + break + if consecutive_failures == self.NO_MAIL_FAILURES_COUNT: + consecutive_msg = ( + "Maximum consecutive tester failures reached (%s).\nNo more emails will be sent." + % consecutive_failures + ) + should_send_email = True + elif consecutive_failures < self.NO_MAIL_FAILURES_COUNT: + consecutive_msg = ( + "Consecutive tester failures: %s (will stop sending reports when reaching %s failures)." + % (consecutive_failures, self.NO_MAIL_FAILURES_COUNT) + ) + should_send_email = True + else: + consecutive_msg = ( + "Too many consecutive tester failures: %s, no email will be sent." + % consecutive_failures + ) + print(consecutive_msg) + html_report += "\n<br/>" + consecutive_msg.replace("\n", "\n<br/>") + if should_send_email: + sender = "support@ubicast.eu" + recipients = get_conf("EMAIL_ADMINS") or "" + system_domain = get_conf("MS_SERVER_NAME") + system_type = "MediaServer" + if system_domain == "mediaserver": + system_domain = get_conf("CM_SERVER_NAME") + system_type = "MirisManager" + if system_domain == "mirismanager": + system_domain = get_conf("MONITOR_SERVER_NAME") + system_type = "Server" + if system_domain == "monitor": + system_type = "-" + if out_of_support: + recipients = recipients.replace("sysadmin@ubicast.eu", "").replace( + ",,", "," + ) + elif get_conf("PREMIUM_SUPPORT") != "0": + system_domain = "[PREMIUM] %s" % system_domain + recipients = recipients.replace("sysadmin@ubicast.eu", "").replace( + ",,", "," + ) + recipients += ",sysadmin+premium@ubicast.eu" + recipients = recipients.strip(",") + if not recipients: + print( + "No recipients defined for email sending. Set a value for EMAIL_ADMINS." + ) + return 1 + boundary = str(uuid.uuid4()) + if get_conf("TESTER_BASE64_ATTACH") != "0": + log_content_encoding = "base64" + log_content = base64.b64encode(log_content.encode("utf-8")).decode() + mail = """From: %(hostname)s <%(sender)s> +To: %(recipients)s +Subject: %(system_domain)s (%(hostname)s) %(system_type)s health report: %(status)s +Mime-Version: 1.0 +Content-type: multipart/related; boundary="%(boundary)s" + +--%(boundary)s +Content-Type: text/html; charset=UTF-8 +Content-transfer-encoding: utf-8 + +<p><b>Date: %(date)s UTC</b></p> +%(report)s + +--%(boundary)s +Content-type: text/plain; name="%(log_name)s"; charset=UTF-8 +Content-disposition: attachment; filename="%(log_name)s" +Content-transfer-encoding: %(log_content_encoding)s + +%(log_content)s""" % dict( + boundary=boundary, + sender=sender, + hostname=hostname, + recipients=recipients, + status=("KO (%s tests failed)" % failures) if failures > 0 else "OK", + date=now.strftime("%Y-%m-%d %H:%M:%S"), + report=html_report, + log_name=log_name, + log_content_encoding=log_content_encoding, + log_content=log_content, + system_domain=system_domain, + system_type=system_type, + ) + p = subprocess.Popen( + ["/usr/sbin/sendmail", "-t"], + stdin=subprocess.PIPE, + stdout=sys.stdout.stream, + stderr=sys.stderr.stream, + ) + p.communicate(input=mail.encode("utf-8")) + if p.returncode != 0: + print("Failed to send email.") + return 1 + else: + print("Email sent to: %s" % recipients) + exit_code = 1 if failures > 0 else 0 + return exit_code + + +if __name__ == "__main__": + Tester() diff --git a/update_envsetup.py b/tests/update_envsetup.py similarity index 90% rename from update_envsetup.py rename to tests/update_envsetup.py index adaf9651..b5b347e0 100755 --- a/update_envsetup.py +++ b/tests/update_envsetup.py @@ -4,11 +4,11 @@ import os import subprocess import sys -import utils +from utilities.config import get_conf if __name__ == "__main__": - branch = utils.get_conf("ENVSETUP_BRANCH") or "stable" + branch = get_conf("ENVSETUP_BRANCH") or "stable" os.chdir(os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))) sys.stdout.write("Updating envsetup: ") sys.stdout.flush() diff --git a/logs/.gitkeep b/tests/utilities/__init__.py similarity index 100% rename from logs/.gitkeep rename to tests/utilities/__init__.py diff --git a/utils_lib/apt.py b/tests/utilities/apt.py similarity index 98% rename from utils_lib/apt.py rename to tests/utilities/apt.py index 02b5e1c4..3d04f059 100644 --- a/utils_lib/apt.py +++ b/tests/utilities/apt.py @@ -6,12 +6,10 @@ A wrapper of apt module that is actually usable. import os -import utils as u - try: import apt except ModuleNotFoundError: - u.warning("apt python module not found") + print("Python apt module not found.") exit(2) diff --git a/utils_lib/commands.py b/tests/utilities/commands.py similarity index 100% rename from utils_lib/commands.py rename to tests/utilities/commands.py diff --git a/utils_lib/config.py b/tests/utilities/config.py similarity index 98% rename from utils_lib/config.py rename to tests/utilities/config.py index d623ce4e..813e73b8 100644 --- a/utils_lib/config.py +++ b/tests/utilities/config.py @@ -24,7 +24,7 @@ def load_conf() -> dict: """ conf = {} - base_dir = str(Path(get_dir(__file__)).parent) + base_dir = str(Path(get_dir(__file__)).parent.parent) files = ( (str(Path(base_dir, DEFAULT_CONF_PATH)), True), (str(Path(base_dir, AUTO_CONF_PATH)), False), diff --git a/utils_lib/logging.py b/tests/utilities/logging.py similarity index 100% rename from utils_lib/logging.py rename to tests/utilities/logging.py diff --git a/utils_lib/network.py b/tests/utilities/network.py similarity index 100% rename from utils_lib/network.py rename to tests/utilities/network.py diff --git a/utils_lib/os.py b/tests/utilities/os.py similarity index 100% rename from utils_lib/os.py rename to tests/utilities/os.py diff --git a/getenvsetup.sh b/tools/getenvsetup.sh similarity index 100% rename from getenvsetup.sh rename to tools/getenvsetup.sh diff --git a/kernels_cleaner.py b/tools/kernels_cleaner.py similarity index 100% rename from kernels_cleaner.py rename to tools/kernels_cleaner.py diff --git a/packer/example.json b/tools/packer/example.json similarity index 100% rename from packer/example.json rename to tools/packer/example.json diff --git a/packer/files/preseed.cfg b/tools/packer/files/preseed.cfg similarity index 100% rename from packer/files/preseed.cfg rename to tools/packer/files/preseed.cfg diff --git a/packer/files/root.cfg b/tools/packer/files/root.cfg similarity index 100% rename from packer/files/root.cfg rename to tools/packer/files/root.cfg diff --git a/packer/files/support.pub b/tools/packer/files/support.pub similarity index 100% rename from packer/files/support.pub rename to tools/packer/files/support.pub diff --git a/packer/scripts/root.sh b/tools/packer/scripts/root.sh similarity index 100% rename from packer/scripts/root.sh rename to tools/packer/scripts/root.sh diff --git a/packer/scripts/upgrade.sh b/tools/packer/scripts/upgrade.sh similarity index 100% rename from packer/scripts/upgrade.sh rename to tools/packer/scripts/upgrade.sh diff --git a/set_app_domain.py b/tools/set_app_domain.py similarity index 86% rename from set_app_domain.py rename to tools/set_app_domain.py index b96b3e13..4e5a7790 100755 --- a/set_app_domain.py +++ b/tools/set_app_domain.py @@ -8,8 +8,19 @@ import re import subprocess import sys -import utils -from utils import log + +def log(text, error=False): + fo = sys.stderr if error else sys.stdout + print(text, file=fo) + fo.flush() + + +def run_cmd(cmd, shell=False): + p = subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8', check=False, shell=shell) + log(p.stdout) + if p.returncode != 0: + raise Exception('Command failed with code %s.' % p.returncode) + return p.returncode class SetAppDomain(): @@ -32,14 +43,6 @@ class SetAppDomain(): if '-h' in args: log(self.USAGE) sys.exit(0) - # Check current dir - root_dir = utils.get_dir(__file__) - if root_dir != '': - os.chdir(root_dir) - self.root_dir = root_dir - # Add to python path - if root_dir not in sys.path: - sys.path.append(root_dir) # Check if force mode is enabled self.force = '-f' in args if self.force: @@ -136,7 +139,8 @@ class SetAppDomain(): with open(path, 'w') as fo: fo.write(new_vhost) log('The configuration file "%s" has been update.' % path) - utils.run_commands(['service nginx restart']) + run_cmd(['nginx', '-t']) + run_cmd(['systemctl', 'restart', 'nginx']) else: log('The configuration file "%s" is already up to date.' % path) @@ -163,7 +167,7 @@ class SetAppDomain(): fo.write(new_hosts) log('The "/etc/hosts" file has been update.') try: - utils.run_commands(['service nscd restart']) + run_cmd(['systemctl', 'restart', 'nscd']) except Exception as nscd_err: log(nscd_err) else: @@ -199,15 +203,16 @@ class SetAppDomain(): log('Assuming that the new url is using HTTPS: "%s"' % new_url) cmds = [ # set site url in site settings - 'python3 %s %s site_url="%s"' % (os.path.join(ms_path, 'scripts', 'mssiteconfig.py'), instance, new_url), + ['python3', os.path.join(ms_path, 'scripts', 'mssiteconfig.py'), instance, 'site_url="%s"' % new_url], # reset all local resources managers - 'python3 %s %s local' % (os.path.join(ms_path, 'scripts', 'reset_service_resources.py'), instance), + ['python3', os.path.join(ms_path, 'scripts', 'reset_service_resources.py'), instance, 'local'], # change configuration of celerity in MS and in workers - 'python3 %s update %s' % (os.path.join(ms_path, 'scripts', 'celerity_config_updater.py'), instance), + ['python3', os.path.join(ms_path, 'scripts', 'celerity_config_updater.py'), 'update', instance], # restart ms - 'mscontroller.py restart -u %s' % instance, + ['mscontroller.py', 'restart', '-u', instance], ] - utils.run_commands(cmds) + for cmd in cmds: + run_cmd(cmd) except Exception as e: log('Unable to set domain in MediaServer database and Celerity config:\n%s' % e) sys.exit(1) @@ -221,11 +226,8 @@ class SetAppDomain(): try: new_url = 'https://%s' % new_domain log('Assuming that the new url is using HTTPS: "%s"' % new_url) - cmds = [ - # set site url in site settings - 'echo \'from skyreach_site.base.models import SiteSettings; ss = SiteSettings.get_singleton(); ss.url = "%s"; ss.save(); print("Site settings saved.")\' | su skyreach -c "python3 /home/skyreach/skyreach_site/manage.py shell -i python"' % new_url, - ] - utils.run_commands(cmds) + # set site url in site settings + run_cmd('echo \'from skyreach_site.base.models import SiteSettings; ss = SiteSettings.get_singleton(); ss.url = "%s"; ss.save(); print("Site settings saved.")\' | su skyreach -c "python3 /home/skyreach/skyreach_site/manage.py shell -i python"' % new_url, shell=True) except Exception as e: log('Unable to set domain in Miris Manager database:\n%s' % e) sys.exit(1) diff --git a/utils.py b/utils.py deleted file mode 100644 index 6c32d0b6..00000000 --- a/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import Path -import sys - -sys.path.append(str(Path(__file__).parent.resolve())) - -from utils_lib import * # noqa: F401, E402, F403 -from utils_lib.commands import * # noqa: F401, E402, F403 -from utils_lib.config import * # noqa: F401, E402, F403 -from utils_lib.logging import * # noqa: F401, E402, F403 -from utils_lib.network import * # noqa: F401, E402, F403 -from utils_lib.os import * # noqa: F401, E402, F403 diff --git a/utils_lib/__init__.py b/utils_lib/__init__.py deleted file mode 100644 index f0c9edeb..00000000 --- a/utils_lib/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python3 - -"""EnvSetup utilities.""" -- GitLab