From 412bab86b64855a40f5acbc9634853997078a46b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Florent=20Thi=C3=A9ry?= <florent.thiery@ubicast.eu>
Date: Fri, 24 Mar 2017 12:27:37 +0100
Subject: [PATCH] create ftp_hotfolder envsetup installer, fixes #20946
---
5.MediaServer/4.FTP_hotfolder/0_setup.py | 72 ++++++++++++++++++
.../4.FTP_hotfolder/create_ftp_account.sh | 4 +
.../4.FTP_hotfolder/on_ftp_upload.py | 76 +++++++++++++++++++
.../4.FTP_hotfolder/pure-ftpd-common | 26 +++++++
.../4.FTP_hotfolder/remove_empty_dirs.py | 45 +++++++++++
global-conf.sh | 10 +++
6 files changed, 233 insertions(+)
create mode 100644 5.MediaServer/4.FTP_hotfolder/0_setup.py
create mode 100755 5.MediaServer/4.FTP_hotfolder/create_ftp_account.sh
create mode 100755 5.MediaServer/4.FTP_hotfolder/on_ftp_upload.py
create mode 100644 5.MediaServer/4.FTP_hotfolder/pure-ftpd-common
create mode 100755 5.MediaServer/4.FTP_hotfolder/remove_empty_dirs.py
diff --git a/5.MediaServer/4.FTP_hotfolder/0_setup.py b/5.MediaServer/4.FTP_hotfolder/0_setup.py
new file mode 100644
index 00000000..c760bedd
--- /dev/null
+++ b/5.MediaServer/4.FTP_hotfolder/0_setup.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+
+import utils
+
+
+def setup(interactive=True):
+ dir_path = utils.get_dir(__file__)
+ pwd_path = '/etc/pure-ftpd/pureftpd.passwd'
+ # Get passwords
+ ftpincoming_users = utils.get_conf('FTP_INCOMING_USERS', '').strip(',').split(',')
+ if not ftpincoming_users:
+ raise Exception('No users specified in the configuration FTP_INCOMING_USERS variable.')
+ for user in ftpincoming_users:
+ if ':' not in user or user.count(':') > 1:
+ raise Exception('Invalid user/pass definition, separator not present or too many detected')
+ # Run commands
+ cmds = [
+ 'apt-get install --yes pure-ftpd python3-unidecode',
+ dict(line='adduser --disabled-login --gecos "" --shell /bin/false ftp', cond='id ftp', cond_neg=True, cond_skip=True),
+ 'cp "%s/create_ftp_account.sh" /usr/local/bin' % (dir_path),
+ 'mkdir -p /home/ftp/storage',
+ 'mkdir -p /home/ftp/storage/incoming',
+ 'mkdir -p /home/ftp/storage/hotfolder',
+ 'chmod -R 775 /home/ftp/storage/incoming',
+ 'chmod -R 775 /home/ftp/storage/hotfolder',
+ # Config
+ 'echo "no" > /etc/pure-ftpd/conf/AllowDotFiles',
+ 'echo "yes" > /etc/pure-ftpd/conf/CallUploadScript',
+ 'echo "yes" > /etc/pure-ftpd/conf/ChrootEveryone',
+ 'echo "yes" > /etc/pure-ftpd/conf/DontResolve',
+ 'echo "no" > /etc/pure-ftpd/conf/PAMAuthentication',
+ # Post upload script
+ 'cp "%s/on_ftp_upload.py" /home/ftp/on_ftp_upload.py' % dir_path,
+ 'chown ftp:ftp /home/ftp/on_ftp_upload.py',
+ 'chmod +x /home/ftp/on_ftp_upload.py',
+ 'pure-uploadscript -p /home/ftp/.on_upload.pid -B -g $(id -g ftp) -r /home/ftp/on_ftp_upload.py -u $(id -u ftp)',
+ 'cp "%s/pure-ftpd-common" /etc/default/pure-ftpd-common.tmp' % dir_path,
+ 'sed "s/UPLOADUID=UID/UPLOADUID=$(id -u ftp)/g" /etc/default/pure-ftpd-common.tmp > /etc/default/pure-ftpd-common.tmp2',
+ 'mv -f /etc/default/pure-ftpd-common.tmp2 /etc/default/pure-ftpd-common.tmp',
+ 'sed "s/UPLOADGID=GID/UPLOADGID=$(id -g ftp)/g" /etc/default/pure-ftpd-common.tmp > /etc/default/pure-ftpd-common.tmp2',
+ 'mv -f /etc/default/pure-ftpd-common.tmp2 /etc/default/pure-ftpd-common',
+ 'rm -f /etc/default/pure-ftpd-common.tmp',
+ 'cp "%s/remove_empty_dirs.py" /etc/cron.hourly/remove_empty_dirs.py' % dir_path,
+ # Create FTP accounts
+ '([ -f "%s" ] || [ -f "%s" ] && cp "%s" "%s") || true' % (pwd_path + '.back', pwd_path, pwd_path, pwd_path + '.back'),
+ '([ -f "%s" ] && mv -f "%s" pureftpd.passwd.tmp) || true' % (pwd_path, pwd_path),
+ ]
+ for ftpuser in ftpincoming_users:
+ login, password = ftpuser.split(':')
+ cmds.extend([
+ 'mkdir -p /home/ftp/storage/incoming/%s' % login,
+ '"%s/create_ftp_account.sh" %s "%s" /home/ftp/storage/%s' % (dir_path, login, password, login),
+ ])
+ cmds.extend([
+ 'chmod -R 775 /home/ftp/storage/incoming',
+ 'chown -R ftp:ftp /home/ftp/storage',
+ 'rm -f pureftpd.passwd.tmp',
+ 'pure-pw mkdb',
+ 'ln -sfn /etc/pure-ftpd/conf/PureDB /etc/pure-ftpd/auth/50puredb',
+ '/etc/init.d/pure-ftpd force-reload',
+ ])
+
+ try:
+ utils.run_commands(cmds)
+ except Exception:
+ raise
+ finally:
+ # Restore password conf if required
+ if os.path.exists('pureftpd.passwd.tmp'):
+ os.rename('pureftpd.passwd.tmp', pwd_path)
diff --git a/5.MediaServer/4.FTP_hotfolder/create_ftp_account.sh b/5.MediaServer/4.FTP_hotfolder/create_ftp_account.sh
new file mode 100755
index 00000000..a04ded37
--- /dev/null
+++ b/5.MediaServer/4.FTP_hotfolder/create_ftp_account.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Usage: create_account.sh username password home
+echo -e "$2\n$2" | pure-pw useradd $1 -u ftp -d $3
diff --git a/5.MediaServer/4.FTP_hotfolder/on_ftp_upload.py b/5.MediaServer/4.FTP_hotfolder/on_ftp_upload.py
new file mode 100755
index 00000000..2b6ee58c
--- /dev/null
+++ b/5.MediaServer/4.FTP_hotfolder/on_ftp_upload.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import logging
+import os
+import shutil
+import sys
+import unicodedata
+# command line
+# pure-uploadscript -p /home/ftp/.on_ftp_upload.pid -B -g 1001 -r /home/ftp/on_ftp_upload.py -u 1001
+
+BASE_DIR = '/home/ftp/storage/'
+INCOMING_DIR = BASE_DIR + 'incoming/'
+DEST_DIR = BASE_DIR + 'hotfolder/'
+ALLOWED_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.'
+
+LOG_FILE = '/home/ftp/on_ftp_upload.log'
+LOG_LEVEL = 'INFO'
+
+
+def clean_name(name):
+ # strip accents and replace non allowed characters
+ return ''.join((c if c in ALLOWED_CHARS else '_') for c in unicodedata.normalize('NFD', name) if unicodedata.category(c) != 'Mn')
+
+
+if __name__ == '__main__':
+ # setup logging
+ logging.basicConfig(
+ filename=LOG_FILE,
+ format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+ level=getattr(logging, LOG_LEVEL),
+ )
+
+ try:
+ logging.debug('Starting script')
+ # see man pure-uploadscript
+ if len(sys.argv) < 2:
+ logging.info('Not enough arguments')
+ sys.exit(1)
+
+ src_path = sys.argv[1]
+ if not src_path.startswith(BASE_DIR):
+ logging.info('File %s will not be moved because it is not in base dir', src_path)
+ sys.exit(0)
+
+ # remove special characters
+ name = os.path.basename(src_path)
+ new_name = clean_name(name)
+ if name != new_name:
+ new_path = os.path.join(os.path.dirname(src_path), new_name)
+ os.rename(src_path, new_path)
+ logging.info('File %s has been renamed to %s', src_path, new_path)
+ src_path = new_path
+
+ # move file
+ if not src_path.startswith(INCOMING_DIR):
+ logging.info('File %s will not be moved because it is not in the incoming dir', src_path)
+ sys.exit(0)
+
+ dest_path = src_path.replace(INCOMING_DIR, DEST_DIR)
+ ftpuser = os.environ.get('UPLOAD_USER')
+ if ftpuser:
+ dest_path = os.path.join(dest_path, ftpuser)
+ else:
+ logging.warning('No ftp user specified, will move into generic folder %s' % dest_path)
+
+ if not os.path.exists(os.path.dirname(dest_path)):
+ os.system('mkdir -p -m 775 "%s"' % os.path.dirname(dest_path))
+
+ logging.info('Moving %s to %s' % (src_path, dest_path))
+ shutil.move(src_path, dest_path)
+ logging.info('Done')
+ except Exception as e:
+ logging.error('Failed to move file %s. Error: %s', src_path, e)
+ sys.exit(1)
+ else:
+ sys.exit(0)
diff --git a/5.MediaServer/4.FTP_hotfolder/pure-ftpd-common b/5.MediaServer/4.FTP_hotfolder/pure-ftpd-common
new file mode 100644
index 00000000..c5d42089
--- /dev/null
+++ b/5.MediaServer/4.FTP_hotfolder/pure-ftpd-common
@@ -0,0 +1,26 @@
+# Configuration for pure-ftpd
+# (this file is sourced by /bin/sh, edit accordingly)
+
+# STANDALONE_OR_INETD
+# valid values are "standalone" and "inetd".
+# Any change here overrides the setting in debconf.
+STANDALONE_OR_INETD=standalone
+
+# VIRTUALCHROOT:
+# whether to use binary with virtualchroot support
+# valid values are "true" or "false"
+# Any change here overrides the setting in debconf.
+VIRTUALCHROOT=false
+
+# UPLOADSCRIPT: if this is set and the daemon is run in standalone mode,
+# pure-uploadscript will also be run to spawn the program given below
+# for handling uploads. see /usr/share/doc/pure-ftpd/README.gz or
+# pure-uploadscript(8)
+
+# example: UPLOADSCRIPT=/usr/local/sbin/uploadhandler.pl
+UPLOADSCRIPT=/home/ftp/on_ftp_upload.py
+
+# if set, pure-uploadscript will spawn $UPLOADSCRIPT running as the
+# given uid and gid
+UPLOADUID=UID
+UPLOADGID=GID
diff --git a/5.MediaServer/4.FTP_hotfolder/remove_empty_dirs.py b/5.MediaServer/4.FTP_hotfolder/remove_empty_dirs.py
new file mode 100755
index 00000000..17c50913
--- /dev/null
+++ b/5.MediaServer/4.FTP_hotfolder/remove_empty_dirs.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+'''
+Script to remove empty dirs from FTP incoming dir.
+'''
+import datetime
+import os
+import shutil
+import sys
+import traceback
+
+
+INCOMING_DIR = '/home/ftp/storage/incoming/'
+DAYS_OLD = 1
+
+
+def _can_be_removed(path):
+ if not os.path.isdir(path):
+ return False
+ for name in os.listdir(path):
+ subpath = os.path.join(path, name)
+ if not _can_be_removed(subpath):
+ return False
+ mtime = os.path.getmtime(path)
+ mtime = datetime.datetime.fromtimestamp(mtime)
+ if mtime < datetime.datetime.now() - datetime.timedelta(days=DAYS_OLD):
+ return True
+ return False
+
+
+if __name__ == '__main__':
+ script_name = os.path.basename(__file__)
+ try:
+ if not os.path.isdir(INCOMING_DIR):
+ print('%s: The FTP incoming dir does not exist (%s).' % (script_name, INCOMING_DIR))
+ sys.exit(1)
+
+ for name in os.listdir(INCOMING_DIR):
+ path = os.path.join(INCOMING_DIR, name)
+ if _can_be_removed(path):
+ shutil.rmtree(path)
+ print('%s: Dir "%s" removed.' % (script_name, path))
+ except Exception:
+ print('%s: Script crashed:\n%s' % (script_name, traceback.format_exc()))
+ sys.exit(1)
diff --git a/global-conf.sh b/global-conf.sh
index aeb74f09..485b5af0 100644
--- a/global-conf.sh
+++ b/global-conf.sh
@@ -110,6 +110,16 @@ HA_LB1_IP='192.168.41.177'
HA_LB2=
HA_LB2_IP=
+# -- FTP --
+# move uploaded files into hotfolder
+# login:pass CSV separated
+#FTP_INCOMING_USERS='ftpuser1:ftppass1,ftpuser2:ftppass2'
+FTP_INCOMING_USERS=
+
+# -- HOTFOLDER --
+# csv-separated
+HOTFOLDERS='/home/ftp/storage/hotfolder'
+
# -- Tester config --
# separate values with commas
TESTER_MS_INSTANCES=
--
GitLab