test_backup.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2017, Florent Thiery
'''
Criticality: Normal
Checks that the server backups are not older than a day.
'''
from datetime import datetime
import imp
import os
import socket
import subprocess
import sys


GREEN = '\033[92m'
RED = '\033[91m'
DEF = '\033[0m'


def test_ssh(ip):
    cmd = 'ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no %s ls /tmp' % ip
    print('Connecting to MediaVault: %s' % cmd)
    try:
        subprocess.check_output(cmd, shell=True, timeout=2)
        print('%sLogged in successfully%s' % (GREEN, DEF))
    except subprocess.CalledProcessError:
        print('%sFailed to login using SSH, run ssh-copy-id %s %s' % (RED, ip, DEF))
        return False
    return True


def test_last_backup_is_recent(server):
    client = socket.gethostname()
    path = '/backup/%s/home/latest' % client
    cmd = 'ssh -o StrictHostKeyChecking=no %s ls -l %s | grep latest' % (server, path)
    status, out = subprocess.getstatusoutput(cmd)
    if status == 0:
        date = out.strip().split(' ')[-1]
        pdate = datetime.strptime(date, '%Y-%m-%d-%H%M%S')
        if (datetime.now() - pdate).days > 2:
            print('Backup is older than 2 days')
            return False
        else:
            print('There is a backup that is less than 2 days old, this is fine')
            return True
    else:
        out = out or 'No output.'
        print('SSH access is not working (code: %s):\n%s' % (status, out))
        return False


def test_backup_space(server):
    cmd = 'ssh -o StrictHostKeyChecking=no %s df -h /backup | tail -n 1' % (server)
    status, out = subprocess.getstatusoutput(cmd)
    if status == 0:
        dev, total, used, free, used_perc, mount = out.strip().split()
        used_perc = int(used_perc.replace('%',''))
        if used_perc > 80:
            print('There is less than 20% of available space for backups')
            return False
        else:
            print('There is %s%% of free space' % (100 - used_perc))
            return True
    else:
        print('Failed to check backup space')
        return False


def check_local_backup(path):
    all_ok = True
    backup_folder = os.path.dirname(path)
    print('Checking %s' % backup_folder)
    latest = os.path.join(backup_folder, 'latest')
    if os.path.exists(latest):
        mtime = os.path.getmtime(latest)
        d = datetime.fromtimestamp(mtime)
        now = datetime.now()
        diff_seconds = (now - d).total_seconds()
        if diff_seconds > 48*3600:
            print('Backup %s is older than 48h (%ih)' % (backup_folder, diff_seconds/3600))
            all_ok = False 
        else:
            print('Backup %s is fine' % backup_folder)
    elif os.path.exists(os.path.join(backup_folder, 'backup.inprogress')):
        print('Backup %s still running' % backup_folder)
        all_ok = False
    else:
        print('Backup %s is not working' % latest)
        all_ok = False
    return all_ok


def check_local_backups(paths):
    all_ok = True
    folders = paths.split(',')
    for f in folders:
        cmd = "find %s -maxdepth 4 -name backup.marker" % f
        status, out = subprocess.getstatusoutput(cmd)
        for bf in out.split('\n'):
            all_ok = min(check_local_backup(bf), all_ok)
    return all_ok


os.chdir(os.path.dirname(__file__))
if os.path.isfile('../utils.py'):
    es_utils = imp.load_source('es_utils', '../utils.py')
    conf = es_utils.load_conf()
    BACKUP_SERVER = conf.get('BACKUP_SERVER')
    LOCAL_BACKUP_FOLDERS = conf.get('LOCAL_BACKUP_FOLDERS')
    if BACKUP_SERVER:
        if not test_ssh(BACKUP_SERVER):
            print('Failed to ssh into backup server')
            sys.exit(1)
        else:
            if not test_last_backup_is_recent(BACKUP_SERVER):
                sys.exit(1)
            else:
                if not test_backup_space(BACKUP_SERVER):
                    sys.exit(1)
                else:
                    sys.exit(0)
    elif LOCAL_BACKUP_FOLDERS:
        sys.exit(check_local_backups(LOCAL_BACKUP_FOLDERS))
    else:
        print('No BACKUP_SERVER defined in config, untestable')
        sys.exit(2)
else:
    print('Unable to load config, untestable')
    sys.exit(2)