#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2017, Florent Thiery ''' Criticality: Normal Checks that the server backups are not older than a day. ''' from datetime import datetime import imp import os import socket import subprocess import sys import glob GREEN = '\033[92m' RED = '\033[91m' DEF = '\033[0m' MAX_AGE_H = 48 def print_red(string): print(RED + string + DEF) def print_green(string): print(GREEN + string + DEF) def test_ssh(ip): cmd = 'ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no %s ls /tmp' % ip print('Connecting to MediaVault: %s' % cmd) try: subprocess.check_output(cmd, shell=True, timeout=2) print('%sLogged in successfully%s' % (GREEN, DEF)) except subprocess.CalledProcessError: print('%sFailed to login using SSH, run ssh-copy-id %s %s' % (RED, ip, DEF)) return False return True def test_last_backup_is_recent(server): client = socket.gethostname() path = '/backup/%s/home/latest' % client cmd = 'ssh -o StrictHostKeyChecking=no %s ls -l %s | grep latest' % (server, path) status, out = subprocess.getstatusoutput(cmd) if status == 0: date = out.strip().split(' ')[-1] pdate = datetime.strptime(date, '%Y-%m-%d-%H%M%S') if (datetime.now() - pdate).days > 2: print('Backup is older than 2 days') return False else: print('There is a backup that is less than 2 days old, this is fine') return True else: out = out or 'No output.' print('SSH access is not working (code: %s):\n%s' % (status, out)) return False def test_backup_space(server=None, path="/backup"): cmd = 'df -h %s | tail -n 1' % path if server: cmd = 'ssh -o StrictHostKeyChecking=no %s ' % server + cmd status, out = subprocess.getstatusoutput(cmd) if status == 0: dev, total, used, free, used_perc, mount = out.strip().split() used_perc = int(used_perc.replace('%', '')) if used_perc > 80: print('There is less than 20% of available space for backups') return False else: print('There is %s%% of free space' % (100 - used_perc)) return True else: print('Failed to check backup space') return False def check_backup_is_incremental(path): dirs = os.listdir(path) dirs.sort() for d in dirs: num_folders = 0 if os.path.isdir(os.path.join(path, d)): media = glob.glob(os.path.join(path, d, "*/msinstance/media/resources/")) for m in media: num_folders += len(os.listdir(m)) print('%s: %s' % (d, num_folders)) if num_folders == 0: print('Folder %s is empty, this indicates non-incremental backups (we are expecting links)' % d) return False return True def check_local_backup(path): all_ok = True backup_folder = os.path.dirname(path) print('Checking %s' % backup_folder) latest = os.path.join(backup_folder, 'latest') if os.path.exists(latest): # resolve symbolic link latest = os.path.realpath(latest) date = os.path.basename(latest) d = datetime.strptime(date, '%Y-%m-%d-%H%M%S') now = datetime.now() diff_seconds = (now - d).total_seconds() if diff_seconds > MAX_AGE_H * 3600: print_red('Backup %s is older than %sh (%ih)' % (backup_folder, MAX_AGE_H, diff_seconds / 3600)) all_ok = False else: print_green('Backup %s is fine' % backup_folder) if not check_backup_is_incremental(backup_folder): all_ok = False elif os.path.exists(os.path.join(backup_folder, 'backup.inprogress')): print_red('Initial backup %s still running' % backup_folder) all_ok = False else: print_red('Backup %s is not working' % latest) all_ok = False return all_ok def check_local_backups(paths): all_ok = True folders = paths.split(',') for f in folders: cmd = "find %s -maxdepth 4 -name backup.marker" % f status, out = subprocess.getstatusoutput(cmd) for bf in out.split('\n'): all_ok = min(check_local_backup(bf), all_ok) all_ok = min(test_backup_space(path=f), all_ok) return all_ok os.chdir(os.path.dirname(__file__)) if os.path.isfile('../utils.py'): es_utils = imp.load_source('es_utils', '../utils.py') conf = es_utils.load_conf() BACKUP_SERVER = conf.get('BACKUP_SERVER') LOCAL_BACKUP_FOLDERS = conf.get('LOCAL_BACKUP_FOLDERS') if BACKUP_SERVER: if not test_ssh(BACKUP_SERVER): print('Failed to ssh into backup server') sys.exit(1) else: if not test_last_backup_is_recent(BACKUP_SERVER): sys.exit(1) else: if not test_backup_space(server=BACKUP_SERVER): sys.exit(1) else: sys.exit(0) elif LOCAL_BACKUP_FOLDERS: sys.exit(not check_local_backups(LOCAL_BACKUP_FOLDERS)) else: print('No BACKUP_SERVER defined in config, untestable') sys.exit(2) else: print('Unable to load config, untestable') sys.exit(2)