test_backup.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2017, Florent Thiery
'''
Criticality: Normal
Checks that the server backups are not older than a day.
'''
from datetime import datetime
import imp
import os
import socket
import subprocess
import sys
import glob

GREEN = '\033[92m'
RED = '\033[91m'
DEF = '\033[0m'

MAX_AGE_H = 48


def print_red(string):
    print(RED + string + DEF)


def print_green(string):
    print(GREEN + string + DEF)


def test_ssh(ip):
    cmd = 'ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no %s ls /tmp' % ip
    print('Connecting to MediaVault: %s' % cmd)
    try:
        subprocess.check_output(cmd, shell=True, timeout=2)
        print('%sLogged in successfully%s' % (GREEN, DEF))
    except subprocess.CalledProcessError:
        print('%sFailed to login using SSH, run ssh-copy-id %s %s' % (RED, ip, DEF))
        return False
    return True


def test_last_backup_is_recent(server):
    client = socket.gethostname()
    path = '/backup/%s/home/latest' % client
    cmd = 'ssh -o StrictHostKeyChecking=no %s ls -l %s | grep latest' % (server, path)
    status, out = subprocess.getstatusoutput(cmd)
    if status == 0:
        date = out.strip().split(' ')[-1]
        pdate = datetime.strptime(date, '%Y-%m-%d-%H%M%S')
        if (datetime.now() - pdate).days > 2:
            print('Backup is older than 2 days')
            return False
        else:
            print('There is a backup that is less than 2 days old, this is fine')
            return True
    else:
        out = out or 'No output.'
        print('SSH access is not working (code: %s):\n%s' % (status, out))
        return False


def test_backup_space(server=None, path="/backup"):
    cmd = 'df -h %s | tail -n 1' % path
    if server:
        cmd = 'ssh -o StrictHostKeyChecking=no %s ' % server + cmd
    status, out = subprocess.getstatusoutput(cmd)
    if status == 0:
        dev, total, used, free, used_perc, mount = out.strip().split()
        used_perc = int(used_perc.replace('%', ''))
        if used_perc > 80:
            print('There is less than 20% of available space for backups')
            return False
        else:
            print('There is %s%% of free space' % (100 - used_perc))
            return True
    else:
        print('Failed to check backup space')
        return False


def check_backup_is_incremental(path):
    dirs = os.listdir(path)
    dirs.sort()

    for d in dirs:
        num_folders = 0
        folder_path = os.path.join(path, d)
        if os.path.isdir(folder_path):
            media = glob.glob(os.path.join(path, d, "*/msinstance/media/resources/"))
            for m in media:
                num_folders += len(os.listdir(m))
            print('%s: %s' % (d, num_folders))
            if num_folders == 0:
                print('Folder %s is empty, this indicates non-incremental backups (we are expecting links)' % folder_path)
                os.rmdir(folder_path)
                print('Folder removed')
                return False
    return True


def check_local_backup(path):
    all_ok = True
    backup_folder = os.path.dirname(path)
    print('Checking %s' % backup_folder)
    latest = os.path.join(backup_folder, 'latest')
    if os.path.exists(latest):
        # resolve symbolic link
        latest = os.path.realpath(latest)
        date = os.path.basename(latest)
        d = datetime.strptime(date, '%Y-%m-%d-%H%M%S')
        now = datetime.now()
        diff_seconds = (now - d).total_seconds()
        if diff_seconds > MAX_AGE_H * 3600:
            print_red('Backup %s is older than %sh (%ih)' % (backup_folder, MAX_AGE_H, diff_seconds / 3600))
            all_ok = False
        else:
            print_green('Backup %s is fine' % backup_folder)
        if not check_backup_is_incremental(backup_folder):
            all_ok = False
    elif os.path.exists(os.path.join(backup_folder, 'backup.inprogress')):
        print_red('Initial backup %s still running' % backup_folder)
        all_ok = False
    else:
        print_red('Backup %s is not working' % latest)
        all_ok = False
    return all_ok


def check_local_backups(paths):
    all_ok = True
    folders = paths.split(',')
    for f in folders:
        cmd = "find %s -maxdepth 4 -name backup.marker" % f
        status, out = subprocess.getstatusoutput(cmd)
        for bf in out.split('\n'):
            all_ok = min(check_local_backup(bf), all_ok)
        all_ok = min(test_backup_space(path=f), all_ok)
    return all_ok


os.chdir(os.path.dirname(__file__))
if os.path.isfile('../utils.py'):
    es_utils = imp.load_source('es_utils', '../utils.py')
    conf = es_utils.load_conf()
    BACKUP_SERVER = conf.get('BACKUP_SERVER')
    LOCAL_BACKUP_FOLDERS = conf.get('LOCAL_BACKUP_FOLDERS')
    if BACKUP_SERVER:
        if not test_ssh(BACKUP_SERVER):
            print('Failed to ssh into backup server')
            sys.exit(1)
        else:
            if not test_last_backup_is_recent(BACKUP_SERVER):
                sys.exit(1)
            else:
                if not test_backup_space(server=BACKUP_SERVER):
                    sys.exit(1)
                else:
                    sys.exit(0)
    elif LOCAL_BACKUP_FOLDERS:
        sys.exit(not check_local_backups(LOCAL_BACKUP_FOLDERS))
    else:
        print('No BACKUP_SERVER defined in config, untestable')
        sys.exit(2)
else:
    print('Unable to load config, untestable')
    sys.exit(2)