Refactoring of MediaWorker test | refs #32998

tests/scripts/test_mediaworker.py

@@ -5,122 +5,26 @@ Criticality: High
 Checks that MediaWorker can be reached using SSH and that it can reach the tasks server
 '''
 from pathlib import Path
-import re
-import subprocess
+import requests
 import sys
 
+try:
+    from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+except ImportError:
+    requests.packages.urllib3.disable_warnings()
+
 sys.path.append(str(Path(__file__).parents[1].resolve()))
 
 # pylint: disable=wrong-import-position
 from utilities import logging as lg  # noqa: E402
 
 
-def check_ssh(ip):
-    cmd = 'ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no root@%s ls /tmp' % ip
-    lg.log('Connecting to MediaWorker:\n%s' % cmd)
-    try:
-        subprocess.check_output(cmd, shell=True, timeout=5)
-        lg.success('Logged in successfully in "%s".' % ip)
-    except subprocess.CalledProcessError:
-        lg.error('Failed to login using SSH, run "ssh-copy-id %s".' % ip)
-        return False
-    except subprocess.TimeoutExpired:
-        try:
-            cmd_port = 'nc -z -w2 {} 22'.format(ip)
-            subprocess.check_output(cmd_port, shell=True, timeout=5)
-        except Exception:
-            cmd_ping = 'ping -c2 -w4 {}'.format(ip)
-            subprocess.check_output(cmd_ping, shell=True, timeout=15)
-        return False
-    return True
-
-
-def check_celerity_connectivity(ip):
-    lg.log('Getting celerity server url.')
-    cmd = 'ssh -t root@%s cat /etc/celerity/config.py' % ip
-    lg.log(cmd)
-    try:
-        d = subprocess.check_output(cmd, shell=True, timeout=5, universal_newlines=True)
-    except subprocess.CalledProcessError:
-        m = None
-    else:
-        m = re.search(r'\nSERVER_URL\s*=\s*([:\/\'\"\-\_\.\w]+)', d)
-    if not m:
-        lg.error('Failed to get celerity tasks server url from configuration in MediaWorker "%s".' % ip)
-        return False
-    server_url = m.groups()[0].strip('"\' ')
-    lg.log('Checking celerity connectivity.')
-    cmd = 'ssh -t root@%s curl -k %s' % (ip, server_url)
-    lg.log(cmd)
-    try:
-        d = subprocess.check_output(cmd, shell=True, timeout=5, universal_newlines=True)
-    except subprocess.CalledProcessError:
-        d = ''
-    if 'Celerity tasks server' in d:
-        lg.success('Successfully reached tasks server from MediaWorker "%s".' % ip)
-        return True
-    lg.error('Failed to reach tasks server from MediaWorker "%s".' % ip)
-    return False
-
-
-def check_celerity_versions(ip):
-    lg.log('Checking that celerity server and worker uses the same version.')
-    try:
-        ms_out = subprocess.check_output('dpkg -s celerity-utils | grep "^Version:"', shell=True, timeout=10, universal_newlines=True)
-        mw_out = subprocess.check_output('ssh -t root@%s dpkg -s celerity-utils | grep "^Version:"' % ip, shell=True, timeout=10, universal_newlines=True)
-    except subprocess.CalledProcessError as e:
-        lg.error('Failed to check celerity version in MediaWorker "%s":\n%s' % ip, e)
-        return False
-    ms_out = (ms_out[len('Version:'):] if ms_out.startswith('Version:') else ms_out).strip()
-    mw_out = (mw_out[len('Version:'):] if mw_out.startswith('Version:') else mw_out).strip()
-    if ms_out != mw_out:
-        lg.error('The celerity version in MediaWorker "%s" is not the same as in MediaServer.\nMediaServer version: \tn%s\nMediaWorker version: \t%s' % (ip, ms_out, mw_out))
-        return False
-    lg.success('The celerity version in MediaWorker "%s" is the same as in MediaServer.\nCurrent celerity version is: %s.' % (ip, ms_out))
-    return True
-
-
-def run_tests(ip):
-    lg.log('Updating envsetup tests on MediaWorker.')
-    cmd = 'ssh -t root@%s /root/envsetup/update_envsetup.py' % ip
-    subprocess.run(cmd, shell=True)
-    lg.log('Running envsetup tests on MediaWorker.')
-    cmd = 'ssh -t root@%s /root/envsetup/tester.py' % ip
-    p = subprocess.run(cmd, shell=True)
-    if p.returncode == 0:
-        subprocess.check_output(cmd, shell=True, timeout=60)
-        lg.success('All tests completed on MediaWorker "%s".' % ip)
-        return True
-    else:
-        lg.error('apt-get update failed on MediaWorker "%s".' % ip)
-        return False
-
-
-def get_remote_workers_ips():
+def check_workers():
     '''
-    Returns the list of IP addresses of all non local celerity workers.
+    Check remote workers using the Celerity API.
     '''
-    ips = set()
-    # get worker IPs
-    try:
-        from celerity_utils import api
-        success, response = api.list_workers('celerity_config_updater')
-        if not success:
-            raise Exception(str(response))
-        for worker in response['workers']:
-            if worker['remote_ip']:
-                ips.add(worker['remote_ip'])
-    except Exception as e:
-        lg.error('Failed to get workers list using celerity API: %s' % e)
-    # remove local IP
-    p = subprocess.run(['ip', 'addr'], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
-    local_ips = re.findall(r'inet ([\d\.]+)/', p.stdout)
-    lg.log('Local IP addresses are: %s.' % local_ips)
-    ips.difference(set(local_ips))
-    return ips
-
-
-def main():
     try:
         import mediaserver
     except ImportError:
@@ -129,31 +33,36 @@ def main():
     else:
         lg.log('MediaServer version: %s.' % mediaserver.__version__)
 
-    all_ok = True
-    tested = False
-    worker_ips = get_remote_workers_ips()
-    for worker_ip in worker_ips:
-        worker_ip = worker_ip.strip()
-        if worker_ip and not worker_ip.startswith('127.'):
-            tested = True
-            if not check_ssh(worker_ip):
-                all_ok = False
+    healthy_list = list()
+    offline_list = list()
+    need_update_list = list()
+    try:
+        from celerity_utils import api
+        success, response = api.list_workers('celerity_config_updater')
+        if not success:
+            raise Exception(str(response))
+        for worker in response['workers']:
+            if not worker.get('online'):
+                offline_list.append(worker['name'])
+            elif not worker.get('up_to_date'):
+                need_update_list.append(worker['name'])
             else:
-                if not check_celerity_connectivity(worker_ip):
-                    all_ok = False
-                if not check_celerity_versions(worker_ip):
-                    all_ok = False
-                # if not run_tests(worker_ip):
-                #     all_ok = False
-    if not tested:
-        lg.log('No remote worker found, skipping test.')
-        return 2
-
-    if not all_ok:
+                healthy_list.append(worker['name'])
+    except Exception as e:
+        lg.error('Failed to get workers list using celerity API: %s' % e)
+        return 1
+    if need_update_list:
+        lg.error('The celerity version in following MediaWorkers is not the same as in MediaServer:\n  %s' % ', '.join(need_update_list))
+        return 1
+    if offline_list:
+        lg.warning('Some MediaWorkers are offline:\n  %s' % ', '.join(offline_list))
+    if not healthy_list:
+        lg.error('No MediaWorker is online and up to date.')
         return 1
+    lg.success('The following MediaWorkers are online and up to date:\n  %s' % ', '.join(healthy_list))
     return 0
 
 
 if __name__ == '__main__':
-    code = main()
+    code = check_workers()
     sys.exit(code)