diff --git a/roles/mediaimport/defaults/main.yml b/roles/mediaimport/defaults/main.yml
index ee7c14b755c8520fb96c6092f07304c11af9e15f..6e90ebc7e208eaa66fa5e3e63fadedc352d99d9a 100644
--- a/roles/mediaimport/defaults/main.yml
+++ b/roles/mediaimport/defaults/main.yml
@@ -5,31 +5,11 @@ mediaimport_users:
passwd: "{{ envsetup_mediaimport_password | d() }}"
mediaimport_packages:
- - clamav
- - mysecureshell
- openssh-server
- - openssl
- - pure-ftpd
- - python3-unidecode
+ - mysecureshell # unreleased version that includes CallbackUpload, packaged by UbiCast
+ - pure-ftpd # must be installed before ubicast-mediaimport so that config can be tweaked by the postinst
+ - ssl-cert # for optionnal FTPS support (the mediaimport postinst will will use the snakeoil certificate for pure-ftpd)
- ubicast-mediaimport
- # required by ansible tasks
- - python3-openssl
-
-mediaimport_pureftpd_config:
- - key: AllowDotFiles
- value: "no"
- - key: CallUploadScript
- value: "yes"
- - key: ChrootEveryone
- value: "yes"
- - key: DontResolve
- value: "yes"
- - key: PAMAuthentication
- value: "yes"
- - key: TLS
- value: "1"
-
-mediaimport_virus_scan_on_upload: false
mediaimport_ms_api_key: "{{ envsetup_ms_api_key | d() }}"
mediaimport_ms_server_name: "{{ envsetup_ms_server_name | d() }}"
diff --git a/roles/mediaimport/files/mediaimport b/roles/mediaimport/files/mediaimport
deleted file mode 100644
index 3294539e8c7c441ecca84e81478fdbbc02e7e7da..0000000000000000000000000000000000000000
--- a/roles/mediaimport/files/mediaimport
+++ /dev/null
@@ -1,6 +0,0 @@
-# purge mediaimport files that are older than 60 days
-0 23 * * * root /usr/bin/find /home/ftp/storage/incoming/ -type f -mtime +60 -delete
-
-# purge empty folders
-0 23 * * * root /usr/bin/find /home/ftp/storage -type d -empty -name thumbnails -delete
-0 23 * * * root /usr/bin/find /home/ftp/storage -type d -empty -name "*20*-*" -delete
diff --git a/roles/mediaimport/files/mediaimport.py b/roles/mediaimport/files/mediaimport.py
deleted file mode 100644
index 0cb2b47a7db80ddc10a4f9b40c7ee0eab35cc445..0000000000000000000000000000000000000000
--- a/roles/mediaimport/files/mediaimport.py
+++ /dev/null
@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import crypt
-import shutil
-import subprocess
-
-BASE_DIR = "/home/ftp/storage"
-INCOMING_DIR = BASE_DIR + "/incoming"
-WATCH_DIR = BASE_DIR + "/watchfolder"
-
-
-def main():
- commands = MediaImport()
-
- parser = argparse.ArgumentParser(prog="mediaimport", description=commands.__doc__)
- subparsers = parser.add_subparsers(title="available commands", dest="command")
- subparsers.required = True
-
- # add command and arguments
- parser_add = subparsers.add_parser("add", help=commands.add_user.__doc__)
- parser_add.add_argument(
- "-u",
- "--user",
- help="username",
- action="store",
- type=commands._new_user,
- required=True,
- )
- parser_add.add_argument(
- "-p", "--passwd", help="password", action="store", type=str, required=True
- )
- parser_add.add_argument(
- "-y", "--yes", action="store_true", help="do not prompt for confirmation"
- )
- parser_add.set_defaults(func=commands.add_user)
-
- # delete command and arguments
- parser_del = subparsers.add_parser("delete", help=commands.del_user.__doc__)
- parser_del.add_argument(
- "-u",
- "--user",
- help="username",
- action="store",
- type=commands._user,
- required=True,
- )
- parser_del.add_argument(
- "-y", "--yes", action="store_true", help="do not prompt for confirmation"
- )
- parser_del.set_defaults(func=commands.del_user)
-
- # list command and arguments
- parser_list = subparsers.add_parser("list", help=commands.list_users.__doc__)
- parser_list.set_defaults(func=commands.list_users)
-
- # parse and run
- args = parser.parse_args()
- args.func(args)
-
-
-class MediaImport:
- """Manage mediaimport users."""
-
- def __init__(self):
- self.users = self._get_users()
-
- def _get_users(self) -> list:
- """Get mysecureshell users list."""
-
- with open("/etc/passwd") as fh:
- passwd = fh.readlines()
-
- return sorted(
- [
- u.split(":")[0]
- for u in passwd
- if u.split(":")[-1].strip() == "/usr/bin/mysecureshell"
- ]
- )
-
- def _confirm(self, message: str = None):
- """Ask for confirmation."""
-
- if message:
- print(message)
- choice = input("Do you want to continue [y/N]? ").lower()
-
- if choice not in ["y", "yes"]:
- print("Exit.")
- exit(0)
-
- def _new_user(self, value: str) -> str:
- """Check that username does not exist."""
-
- if value in self.users:
- raise argparse.ArgumentTypeError(f"{value} already exists")
-
- return value
-
- def _user(self, value: str) -> str:
- """Check that username exists."""
-
- if value not in self.users:
- raise argparse.ArgumentTypeError(f"{value} does not exists")
-
- return value
-
- def add_user(self, args: argparse.Namespace):
- """add an user"""
-
- username = args.user
- password = args.passwd
-
- if not args.yes:
- self._confirm(f"MediaImport user '{username}' will be created.")
-
- # create user
- subprocess.Popen(
- [
- "useradd",
- "-b",
- INCOMING_DIR,
- "-m",
- "-p",
- crypt.crypt(password),
- "-s",
- "/usr/bin/mysecureshell",
- "-U",
- username,
- ],
- stdout=subprocess.DEVNULL,
- )
- print(f"User {username} created, adjust /etc/mediaserver/mediaimport.json and restart the service:\nsystemctl restart mediaimport")
-
- def del_user(self, args: argparse.Namespace):
- """delete an user"""
-
- username = args.user
- paths = [f"{INCOMING_DIR}/{username}", f"{WATCH_DIR}/{username}"]
-
- if not args.yes:
- self._confirm(f"MediaImport user '{username}' data will be deleted.")
-
- # remove user
- subprocess.Popen(
- ["userdel", "-f", "-r", username],
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- )
-
- # remove user's folders
- for path in paths:
- shutil.rmtree(path, ignore_errors=True)
- print(f"User {username} deleted, adjust /etc/mediaserver/mediaimport.json and restart the service:\nsystemctl restart mediaimport")
-
- def list_users(self, args: argparse.Namespace):
- """list existing users"""
-
- if len(self.users):
- print("\n".join(self.users))
-
-
-if __name__ == "__main__":
- main()
diff --git a/roles/mediaimport/files/on-upload b/roles/mediaimport/files/on-upload
deleted file mode 100755
index 770286dd32b194ba8c3e4c538fec4faced985db2..0000000000000000000000000000000000000000
Binary files a/roles/mediaimport/files/on-upload and /dev/null differ
diff --git a/roles/mediaimport/files/on-upload.go b/roles/mediaimport/files/on-upload.go
deleted file mode 100644
index b4d27ecf5724da51c3af4c2fe1dc470661ebe518..0000000000000000000000000000000000000000
--- a/roles/mediaimport/files/on-upload.go
+++ /dev/null
@@ -1,141 +0,0 @@
-package main
-
-import (
- "log"
- "os"
- "os/exec"
- "path/filepath"
- "strings"
- "unicode"
-
- "github.com/jessevdk/go-flags"
- "golang.org/x/text/transform"
- "golang.org/x/text/unicode/norm"
-)
-
-const (
- baseDir = "/home/ftp/storage"
- incomingDir = baseDir + "/incoming"
- watchDir = baseDir + "/watchfolder"
- quarantineDir = baseDir + "/quarantine"
-)
-
-func setPermissions(path string) error {
- stat, err := os.Stat(path)
- if err != nil {
- return err
- }
-
- switch mode := stat.Mode(); {
- case mode.IsDir():
- if err := os.Chmod(path, 0755); err != nil {
- return err
- }
- case mode.IsRegular():
- if err := os.Chmod(path, 0644); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func cleanName(filename string) string {
- // normalize
- isMn := func(r rune) bool {
- return unicode.Is(unicode.Mn, r)
- }
- t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
- cleanedName, _, _ := transform.String(t, filename)
-
- // replace non allowed characters
- allowedChars := strings.Split("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.", "")
- for _, filenameChar := range strings.Split(cleanedName, "") {
- flagged := false
- for _, allowedChar := range allowedChars {
- if filenameChar == allowedChar {
- flagged = true
- }
- }
- // if not in allowed list replace by underscore
- if !flagged {
- cleanedName = strings.Replace(cleanedName, filenameChar, "_", 1)
- }
- }
-
- return cleanedName
-}
-
-func virusScan(path string) error {
- // will move file into quarantine directory if infected
- cmd := exec.Command(
- "/usr/bin/clamscan",
- "--quiet",
- "--infected",
- "--recursive",
- "--move="+quarantineDir,
- "--max-scantime=600000", // 10 minutes
- "--max-filesize=4000M",
- "--max-scansize=4000M",
- "--max-files=200",
- "--max-recursion=6",
- "--max-dir-recursion=6",
- path,
- )
- err := cmd.Run()
-
- return err
-}
-
-func main() {
- var opts struct {
- Scan bool `short:"s" long:"scan-virus" description:"Scan file for virus"`
- Args struct {
- SrcPaths []string `positional-arg-name:"path" required:"yes" description:"Paths of uploaded files"`
- } `positional-args:"yes"`
- }
-
- if _, err := flags.Parse(&opts); err != nil {
- os.Exit(1)
- }
-
- for _, srcPath := range opts.Args.SrcPaths {
- // check that file is into incoming folder
- if !strings.HasPrefix(srcPath, baseDir) {
- log.Fatalln("file not in base dir (" + baseDir + "): " + srcPath)
- }
-
- // ensure permissions are correct
- if err := setPermissions(srcPath); err != nil {
- log.Fatalln(err)
- }
-
- // scan for virus if enabled
- if opts.Scan {
- if err := os.MkdirAll(quarantineDir, 0775); err != nil {
- log.Fatalln(err)
- }
- if err := virusScan(srcPath); err != nil {
- log.Fatalln(err)
- }
- }
-
- // cleanup and set destination path
- srcDir, srcFile := filepath.Split(srcPath)
- dstFile := cleanName(srcFile)
- dstDir := strings.ReplaceAll(srcDir, incomingDir, watchDir)
- dstPath := dstDir + dstFile
-
- // create destination directory
- if err := os.MkdirAll(dstDir, 0775); err != nil {
- log.Fatalln(err)
- }
-
- // move file into watchfolder
- if err := os.Rename(srcPath, dstPath); err != nil {
- log.Fatalln(err)
- }
-
- log.Println(srcPath + " moved to " + dstPath)
- }
-}
diff --git a/roles/mediaimport/handlers/main.yml b/roles/mediaimport/handlers/main.yml
index b87528a150512ae485d9f60c64083572a5953062..60e664ba04be31637362603611e5959415a35aff 100644
--- a/roles/mediaimport/handlers/main.yml
+++ b/roles/mediaimport/handlers/main.yml
@@ -29,8 +29,4 @@
name: fail2ban
state: restarted
-- name: sftp-verif
- ansible.builtin.command:
- cmd: timeout 30 sftp-verif
-
...
diff --git a/roles/mediaimport/tasks/main.yml b/roles/mediaimport/tasks/main.yml
index 528c843d876390dd280672549eb5cb0929dda011..9b79be8bdb06d1bdb5cb6da08d00730f97a88225 100644
--- a/roles/mediaimport/tasks/main.yml
+++ b/roles/mediaimport/tasks/main.yml
@@ -8,38 +8,17 @@
## USERS
-- name: create ftp folders
- loop:
- - /home/ftp/storage/incoming
- - /home/ftp/storage/watchfolder
- ansible.builtin.file:
- path: "{{ item }}"
- state: directory
- mode: '755'
-
-- name: deploy users management script
- ansible.builtin.copy:
- src: files/mediaimport.py
- dest: /usr/local/bin/mediaimport
- mode: '755'
-
- name: create users
loop: "{{ mediaimport_users }}"
when:
- item.name | d(false)
- item.passwd | d(false)
no_log: true
- ansible.builtin.command: mediaimport add --yes --user {{ item.name }} --passwd {{ item.passwd }}
+ ansible.builtin.command: /usr/bin/mediaimportctl.py add --yes --user {{ item.name }} --passwd {{ item.passwd }}
args:
creates: /home/ftp/storage/incoming/{{ item.name }}
-- name: deploy on-upload script with setuid
- ansible.builtin.copy:
- src: files/on-upload
- dest: /home/ftp/on-upload
- mode: 04755
-
-## MYSECURESHELL
+## MEDIAIMPORT
- name: enable password login for ssh
notify: restart sshd
@@ -48,97 +27,6 @@
regexp: "^PasswordAuthentication no"
replace: "#PasswordAuthentication yes"
-- name: set the setuid on mysecureshell
- ansible.builtin.file:
- path: /usr/bin/mysecureshell
- mode: 04755
-
-- name: configure mysecureshell
- notify:
- - restart mysecureshell
- - sftp-verif
- ansible.builtin.template:
- src: sftp_config.j2
- dest: /etc/ssh/sftp_config
- mode: '644'
-
-## PURE-FTPD
-
-- name: set pure-ftpd default config
- notify: restart pure-ftpd
- ansible.builtin.copy:
- dest: /etc/default/pure-ftpd-common
- mode: '644'
- content: |
- STANDALONE_OR_INETD=standalone
- VIRTUALCHROOT=false
- UPLOADSCRIPT="/home/ftp/on-upload{% if mediaimport_virus_scan_on_upload %} --scan-virus{% endif %}"
- UPLOADUID=0
- UPLOADGID=0
-
-- name: configure pure-ftpd
- notify: restart pure-ftpd
- loop: "{{ mediaimport_pureftpd_config }}"
- ansible.builtin.copy:
- dest: /etc/pure-ftpd/conf/{{ item.key }}
- content: "{{ item.value }}"
- mode: '644'
-
-## PURE-FTPD CERTIFICATES
-
-- name: create certificate directory
- ansible.builtin.file:
- path: /etc/ssl/{{ ansible_fqdn }}
- state: directory
- mode: '755'
-
-- name: generate an private key
- register: mediaimport_privkey
- community.crypto.openssl_privatekey:
- path: /etc/ssl/{{ ansible_fqdn }}/key.pem
- mode: '600'
-
-- name: generate an csr
- when: mediaimport_privkey is changed # noqa no-handler
- register: mediaimport_csr
- openssl_csr:
- path: /etc/ssl/{{ ansible_fqdn }}/csr.pem
- privatekey_path: /etc/ssl/{{ ansible_fqdn }}/key.pem
- common_name: "{{ ansible_fqdn }}"
- mode: '600'
-
-- name: generate a self-signed certificate
- when: mediaimport_csr is changed # noqa no-handler
- register: mediaimport_cert
- openssl_certificate:
- path: /etc/ssl/{{ ansible_fqdn }}/cert.pem
- privatekey_path: /etc/ssl/{{ ansible_fqdn }}/key.pem
- csr_path: /etc/ssl/{{ ansible_fqdn }}/csr.pem
- provider: selfsigned
- mode: '600'
-
-- name: concatenate key and certificate
- when: mediaimport_cert is changed # noqa no-handler
- notify: restart pure-ftpd
- ansible.builtin.shell: >
- cat /etc/ssl/{{ ansible_fqdn }}/key.pem /etc/ssl/{{ ansible_fqdn }}/cert.pem > /etc/ssl/private/pure-ftpd.pem;
- chmod 600 /etc/ssl/private/pure-ftpd.pem;
-
-- name: generate dhparams
- notify: restart pure-ftpd
- openssl_dhparam:
- path: /etc/ssl/private/pure-ftpd-dhparams.pem
- size: 1024
- mode: '600'
-
-## MEDIAIMPORT
-
-- name: setup cron job
- ansible.builtin.copy:
- src: files/mediaimport
- dest: /etc/cron.d/mediaimport
- mode: '644'
-
- name: configure mediaimport
when:
- mediaimport_ms_api_key | d(false)
@@ -155,6 +43,11 @@
name: mediaimport
enabled: true
+- name: enable mediaimport-cleanup timer
+ ansible.builtin.systemd:
+ name: mediaimport-cleanup.timer
+ enabled: true
+
# FAIL2BAN
- name: deploy fail2ban jail
diff --git a/roles/mediaimport/templates/sftp_config.j2 b/roles/mediaimport/templates/sftp_config.j2
deleted file mode 100644
index b33a786d8fd2b3029536244801a3a1c076859b2d..0000000000000000000000000000000000000000
--- a/roles/mediaimport/templates/sftp_config.j2
+++ /dev/null
@@ -1,26 +0,0 @@
-## MySecureShell Configuration File
-# To get more informations on all possible options, please look at the doc:
-# http://mysecureshell.readthedocs.org
-
-#Default rules for everybody
-<Default>
- GlobalDownload 50k
- GlobalUpload 0
- Download 5k
- Upload 0
- StayAtHome true
- VirtualChroot true
- LimitConnection 100
- LimitConnectionByUser 2
- LimitConnectionByIP 10
- Home /home/ftp/storage/incoming/$USER
- CallbackUpload "/home/ftp/on-upload{% if mediaimport_virus_scan_on_upload %} --scan-virus{% endif %} /home/ftp/storage/incoming/$USER$LAST_FILE_PATH"
- IdleTimeOut 5m
- ResolveIP false
- HideNoAccess true
- DefaultRights 0640 0750
- ShowLinksAsLinks false
- LogFile /var/log/sftp-server.log
- LogLevel 6
- LogSyslog true
-</Default>