create ftp_hotfolder envsetup installer, fixes #20946

5.MediaServer/4.FTP_hotfolder/0_setup.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+
+import utils
+
+
+def setup(interactive=True):
+    dir_path = utils.get_dir(__file__)
+    pwd_path = '/etc/pure-ftpd/pureftpd.passwd'
+    # Get passwords
+    ftpincoming_users = utils.get_conf('FTP_INCOMING_USERS', '').strip(',').split(',')
+    if not ftpincoming_users:
+        raise Exception('No users specified in the configuration FTP_INCOMING_USERS variable.')
+    for user in ftpincoming_users:
+        if ':' not in user or user.count(':') > 1:
+            raise Exception('Invalid user/pass definition, separator not present or too many detected')
+    # Run commands
+    cmds = [
+        'apt-get install --yes pure-ftpd python3-unidecode',
+        dict(line='adduser --disabled-login --gecos "" --shell /bin/false ftp', cond='id ftp', cond_neg=True, cond_skip=True),
+        'cp "%s/create_ftp_account.sh" /usr/local/bin' % (dir_path),
+        'mkdir -p /home/ftp/storage',
+        'mkdir -p /home/ftp/storage/incoming',
+        'mkdir -p /home/ftp/storage/hotfolder',
+        'chmod -R 775 /home/ftp/storage/incoming',
+        'chmod -R 775 /home/ftp/storage/hotfolder',
+        # Config
+        'echo "no" > /etc/pure-ftpd/conf/AllowDotFiles',
+        'echo "yes" > /etc/pure-ftpd/conf/CallUploadScript',
+        'echo "yes" > /etc/pure-ftpd/conf/ChrootEveryone',
+        'echo "yes" > /etc/pure-ftpd/conf/DontResolve',
+        'echo "no" > /etc/pure-ftpd/conf/PAMAuthentication',
+        # Post upload script
+        'cp "%s/on_ftp_upload.py" /home/ftp/on_ftp_upload.py' % dir_path,
+        'chown ftp:ftp /home/ftp/on_ftp_upload.py',
+        'chmod +x /home/ftp/on_ftp_upload.py',
+        'pure-uploadscript -p /home/ftp/.on_upload.pid -B -g $(id -g ftp) -r /home/ftp/on_ftp_upload.py -u $(id -u ftp)',
+        'cp "%s/pure-ftpd-common" /etc/default/pure-ftpd-common.tmp' % dir_path,
+        'sed "s/UPLOADUID=UID/UPLOADUID=$(id -u ftp)/g" /etc/default/pure-ftpd-common.tmp > /etc/default/pure-ftpd-common.tmp2',
+        'mv -f /etc/default/pure-ftpd-common.tmp2 /etc/default/pure-ftpd-common.tmp',
+        'sed "s/UPLOADGID=GID/UPLOADGID=$(id -g ftp)/g" /etc/default/pure-ftpd-common.tmp > /etc/default/pure-ftpd-common.tmp2',
+        'mv -f /etc/default/pure-ftpd-common.tmp2 /etc/default/pure-ftpd-common',
+        'rm -f /etc/default/pure-ftpd-common.tmp',
+        'cp "%s/remove_empty_dirs.py" /etc/cron.hourly/remove_empty_dirs.py' % dir_path,
+        # Create FTP accounts
+        '([ -f "%s" ] || [ -f "%s" ] && cp "%s" "%s") || true' % (pwd_path + '.back', pwd_path, pwd_path, pwd_path + '.back'),
+        '([ -f "%s" ] && mv -f "%s" pureftpd.passwd.tmp) || true' % (pwd_path, pwd_path),
+    ]
+    for ftpuser in ftpincoming_users:
+        login, password = ftpuser.split(':')
+        cmds.extend([
+            'mkdir -p /home/ftp/storage/incoming/%s' % login,
+            '"%s/create_ftp_account.sh" %s "%s" /home/ftp/storage/%s' % (dir_path, login, password, login),
+        ])
+    cmds.extend([
+        'chmod -R 775 /home/ftp/storage/incoming',
+        'chown -R ftp:ftp /home/ftp/storage',
+        'rm -f pureftpd.passwd.tmp',
+        'pure-pw mkdb',
+        'ln -sfn /etc/pure-ftpd/conf/PureDB /etc/pure-ftpd/auth/50puredb',
+        '/etc/init.d/pure-ftpd force-reload',
+    ])
+
+    try:
+        utils.run_commands(cmds)
+    except Exception:
+        raise
+    finally:
+        # Restore password conf if required
+        if os.path.exists('pureftpd.passwd.tmp'):
+            os.rename('pureftpd.passwd.tmp', pwd_path)

5.MediaServer/4.FTP_hotfolder/create_ftp_account.sh

+#!/bin/bash
+
+# Usage: create_account.sh username password home
+echo -e "$2\n$2" | pure-pw useradd $1 -u ftp -d $3

5.MediaServer/4.FTP_hotfolder/on_ftp_upload.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import logging
+import os
+import shutil
+import sys
+import unicodedata
+# command line
+# pure-uploadscript -p /home/ftp/.on_ftp_upload.pid -B -g 1001 -r /home/ftp/on_ftp_upload.py -u 1001
+
+BASE_DIR = '/home/ftp/storage/'
+INCOMING_DIR = BASE_DIR + 'incoming/'
+DEST_DIR = BASE_DIR + 'hotfolder/'
+ALLOWED_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.'
+
+LOG_FILE = '/home/ftp/on_ftp_upload.log'
+LOG_LEVEL = 'INFO'
+
+
+def clean_name(name):
+    # strip accents and replace non allowed characters
+    return ''.join((c if c in ALLOWED_CHARS else '_') for c in unicodedata.normalize('NFD', name) if unicodedata.category(c) != 'Mn')
+
+
+if __name__ == '__main__':
+    # setup logging
+    logging.basicConfig(
+        filename=LOG_FILE,
+        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+        level=getattr(logging, LOG_LEVEL),
+    )
+
+    try:
+        logging.debug('Starting script')
+        # see man pure-uploadscript
+        if len(sys.argv) < 2:
+            logging.info('Not enough arguments')
+            sys.exit(1)
+
+        src_path = sys.argv[1]
+        if not src_path.startswith(BASE_DIR):
+            logging.info('File %s will not be moved because it is not in base dir', src_path)
+            sys.exit(0)
+
+        # remove special characters
+        name = os.path.basename(src_path)
+        new_name = clean_name(name)
+        if name != new_name:
+            new_path = os.path.join(os.path.dirname(src_path), new_name)
+            os.rename(src_path, new_path)
+            logging.info('File %s has been renamed to %s', src_path, new_path)
+            src_path = new_path
+
+        # move file
+        if not src_path.startswith(INCOMING_DIR):
+            logging.info('File %s will not be moved because it is not in the incoming dir', src_path)
+            sys.exit(0)
+
+        dest_path = src_path.replace(INCOMING_DIR, DEST_DIR)
+        ftpuser = os.environ.get('UPLOAD_USER')
+        if ftpuser:
+            dest_path = os.path.join(dest_path, ftpuser)
+        else:
+            logging.warning('No ftp user specified, will move into generic folder %s' % dest_path)
+
+        if not os.path.exists(os.path.dirname(dest_path)):
+            os.system('mkdir -p -m 775 "%s"' % os.path.dirname(dest_path))
+
+        logging.info('Moving %s to %s' % (src_path, dest_path))
+        shutil.move(src_path, dest_path)
+        logging.info('Done')
+    except Exception as e:
+        logging.error('Failed to move file %s. Error: %s', src_path, e)
+        sys.exit(1)
+    else:
+        sys.exit(0)

5.MediaServer/4.FTP_hotfolder/pure-ftpd-common

+# Configuration for pure-ftpd
+# (this file is sourced by /bin/sh, edit accordingly)
+
+# STANDALONE_OR_INETD
+# valid values are "standalone" and "inetd".
+# Any change here overrides the setting in debconf.
+STANDALONE_OR_INETD=standalone
+
+# VIRTUALCHROOT:
+# whether to use binary with virtualchroot support
+# valid values are "true" or "false" 
+# Any change here overrides the setting in debconf.
+VIRTUALCHROOT=false
+
+# UPLOADSCRIPT: if this is set and the daemon is run in standalone mode,
+# pure-uploadscript will also be run to spawn the program given below
+# for handling uploads. see /usr/share/doc/pure-ftpd/README.gz or
+# pure-uploadscript(8)
+
+# example: UPLOADSCRIPT=/usr/local/sbin/uploadhandler.pl
+UPLOADSCRIPT=/home/ftp/on_ftp_upload.py
+
+# if set, pure-uploadscript will spawn $UPLOADSCRIPT running as the
+# given uid and gid
+UPLOADUID=UID
+UPLOADGID=GID

5.MediaServer/4.FTP_hotfolder/remove_empty_dirs.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+'''
+Script to remove empty dirs from FTP incoming dir.
+'''
+import datetime
+import os
+import shutil
+import sys
+import traceback
+
+
+INCOMING_DIR = '/home/ftp/storage/incoming/'
+DAYS_OLD = 1
+
+
+def _can_be_removed(path):
+    if not os.path.isdir(path):
+        return False
+    for name in os.listdir(path):
+        subpath = os.path.join(path, name)
+        if not _can_be_removed(subpath):
+            return False
+    mtime = os.path.getmtime(path)
+    mtime = datetime.datetime.fromtimestamp(mtime)
+    if mtime < datetime.datetime.now() - datetime.timedelta(days=DAYS_OLD):
+        return True
+    return False
+
+
+if __name__ == '__main__':
+    script_name = os.path.basename(__file__)
+    try:
+        if not os.path.isdir(INCOMING_DIR):
+            print('%s: The FTP incoming dir does not exist (%s).' % (script_name, INCOMING_DIR))
+            sys.exit(1)
+
+        for name in os.listdir(INCOMING_DIR):
+            path = os.path.join(INCOMING_DIR, name)
+            if _can_be_removed(path):
+                shutil.rmtree(path)
+                print('%s: Dir "%s" removed.' % (script_name, path))
+    except Exception:
+        print('%s: Script crashed:\n%s' % (script_name, traceback.format_exc()))
+        sys.exit(1)

global-conf.sh

@@ -110,6 +110,16 @@ HA_LB1_IP='192.168.41.177'
 HA_LB2=
 HA_LB2_IP=
 
+# -- FTP --
+# move uploaded files into hotfolder 
+# login:pass CSV separated
+#FTP_INCOMING_USERS='ftpuser1:ftppass1,ftpuser2:ftppass2'
+FTP_INCOMING_USERS=
+
+# -- HOTFOLDER --
+# csv-separated
+HOTFOLDERS='/home/ftp/storage/hotfolder' 
+
 # -- Tester config --
 # separate values with commas
 TESTER_MS_INSTANCES=