Merge branch 't36100-cleanup-mediaimport' into 'main'

move most mediaimport-related items into debian package refs #36100

See merge request sys/ansible-public!22

roles/mediaimport/defaults/main.yml

@@ -5,31 +5,11 @@ mediaimport_users:
     passwd: "{{ envsetup_mediaimport_password | d() }}"
 
 mediaimport_packages:
-  - clamav
-  - mysecureshell
   - openssh-server
-  - openssl
-  - pure-ftpd
-  - python3-unidecode
+  - mysecureshell  # unreleased version that includes CallbackUpload, packaged by UbiCast
+  - pure-ftpd  # must be installed before ubicast-mediaimport so that config can be tweaked by the postinst
+  - ssl-cert  # for optionnal FTPS support (the mediaimport postinst will will use the snakeoil certificate for pure-ftpd)
   - ubicast-mediaimport
-  # required by ansible tasks
-  - python3-openssl
-
-mediaimport_pureftpd_config:
-  - key: AllowDotFiles
-    value: "no"
-  - key: CallUploadScript
-    value: "yes"
-  - key: ChrootEveryone
-    value: "yes"
-  - key: DontResolve
-    value: "yes"
-  - key: PAMAuthentication
-    value: "yes"
-  - key: TLS
-    value: "1"
-
-mediaimport_virus_scan_on_upload: false
 
 mediaimport_ms_api_key: "{{ envsetup_ms_api_key | d() }}"
 mediaimport_ms_server_name: "{{ envsetup_ms_server_name | d() }}"

roles/mediaimport/files/mediaimport

-# purge mediaimport files that are older than 60 days
-0 23 * * * root /usr/bin/find /home/ftp/storage/incoming/ -type f -mtime +60 -delete
-
-# purge empty folders
-0 23 * * * root /usr/bin/find /home/ftp/storage -type d -empty -name thumbnails -delete
-0 23 * * * root /usr/bin/find /home/ftp/storage -type d -empty -name "*20*-*" -delete

roles/mediaimport/files/mediaimport.py

-#!/usr/bin/env python3
-
-import argparse
-import crypt
-import shutil
-import subprocess
-
-BASE_DIR = "/home/ftp/storage"
-INCOMING_DIR = BASE_DIR + "/incoming"
-WATCH_DIR = BASE_DIR + "/watchfolder"
-
-
-def main():
-    commands = MediaImport()
-
-    parser = argparse.ArgumentParser(prog="mediaimport", description=commands.__doc__)
-    subparsers = parser.add_subparsers(title="available commands", dest="command")
-    subparsers.required = True
-
-    # add command and arguments
-    parser_add = subparsers.add_parser("add", help=commands.add_user.__doc__)
-    parser_add.add_argument(
-        "-u",
-        "--user",
-        help="username",
-        action="store",
-        type=commands._new_user,
-        required=True,
-    )
-    parser_add.add_argument(
-        "-p", "--passwd", help="password", action="store", type=str, required=True
-    )
-    parser_add.add_argument(
-        "-y", "--yes", action="store_true", help="do not prompt for confirmation"
-    )
-    parser_add.set_defaults(func=commands.add_user)
-
-    # delete command and arguments
-    parser_del = subparsers.add_parser("delete", help=commands.del_user.__doc__)
-    parser_del.add_argument(
-        "-u",
-        "--user",
-        help="username",
-        action="store",
-        type=commands._user,
-        required=True,
-    )
-    parser_del.add_argument(
-        "-y", "--yes", action="store_true", help="do not prompt for confirmation"
-    )
-    parser_del.set_defaults(func=commands.del_user)
-
-    # list command and arguments
-    parser_list = subparsers.add_parser("list", help=commands.list_users.__doc__)
-    parser_list.set_defaults(func=commands.list_users)
-
-    # parse and run
-    args = parser.parse_args()
-    args.func(args)
-
-
-class MediaImport:
-    """Manage mediaimport users."""
-
-    def __init__(self):
-        self.users = self._get_users()
-
-    def _get_users(self) -> list:
-        """Get mysecureshell users list."""
-
-        with open("/etc/passwd") as fh:
-            passwd = fh.readlines()
-
-        return sorted(
-            [
-                u.split(":")[0]
-                for u in passwd
-                if u.split(":")[-1].strip() == "/usr/bin/mysecureshell"
-            ]
-        )
-
-    def _confirm(self, message: str = None):
-        """Ask for confirmation."""
-
-        if message:
-            print(message)
-        choice = input("Do you want to continue [y/N]? ").lower()
-
-        if choice not in ["y", "yes"]:
-            print("Exit.")
-            exit(0)
-
-    def _new_user(self, value: str) -> str:
-        """Check that username does not exist."""
-
-        if value in self.users:
-            raise argparse.ArgumentTypeError(f"{value} already exists")
-
-        return value
-
-    def _user(self, value: str) -> str:
-        """Check that username exists."""
-
-        if value not in self.users:
-            raise argparse.ArgumentTypeError(f"{value} does not exists")
-
-        return value
-
-    def add_user(self, args: argparse.Namespace):
-        """add an user"""
-
-        username = args.user
-        password = args.passwd
-
-        if not args.yes:
-            self._confirm(f"MediaImport user '{username}' will be created.")
-
-        # create user
-        subprocess.Popen(
-            [
-                "useradd",
-                "-b",
-                INCOMING_DIR,
-                "-m",
-                "-p",
-                crypt.crypt(password),
-                "-s",
-                "/usr/bin/mysecureshell",
-                "-U",
-                username,
-            ],
-            stdout=subprocess.DEVNULL,
-        )
-        print(f"User {username} created, adjust /etc/mediaserver/mediaimport.json and restart the service:\nsystemctl restart mediaimport")
-
-    def del_user(self, args: argparse.Namespace):
-        """delete an user"""
-
-        username = args.user
-        paths = [f"{INCOMING_DIR}/{username}", f"{WATCH_DIR}/{username}"]
-
-        if not args.yes:
-            self._confirm(f"MediaImport user '{username}' data will be deleted.")
-
-        # remove user
-        subprocess.Popen(
-            ["userdel", "-f", "-r", username],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-        )
-
-        # remove user's folders
-        for path in paths:
-            shutil.rmtree(path, ignore_errors=True)
-        print(f"User {username} deleted, adjust /etc/mediaserver/mediaimport.json and restart the service:\nsystemctl restart mediaimport")
-
-    def list_users(self, args: argparse.Namespace):
-        """list existing users"""
-
-        if len(self.users):
-            print("\n".join(self.users))
-
-
-if __name__ == "__main__":
-    main()

roles/mediaimport/files/on-upload.go

-package main
-
-import (
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"unicode"
-
-	"github.com/jessevdk/go-flags"
-	"golang.org/x/text/transform"
-	"golang.org/x/text/unicode/norm"
-)
-
-const (
-	baseDir       = "/home/ftp/storage"
-	incomingDir   = baseDir + "/incoming"
-	watchDir      = baseDir + "/watchfolder"
-	quarantineDir = baseDir + "/quarantine"
-)
-
-func setPermissions(path string) error {
-	stat, err := os.Stat(path)
-	if err != nil {
-		return err
-	}
-
-	switch mode := stat.Mode(); {
-	case mode.IsDir():
-		if err := os.Chmod(path, 0755); err != nil {
-			return err
-		}
-	case mode.IsRegular():
-		if err := os.Chmod(path, 0644); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func cleanName(filename string) string {
-	// normalize
-	isMn := func(r rune) bool {
-		return unicode.Is(unicode.Mn, r)
-	}
-	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
-	cleanedName, _, _ := transform.String(t, filename)
-
-	// replace non allowed characters
-	allowedChars := strings.Split("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.", "")
-	for _, filenameChar := range strings.Split(cleanedName, "") {
-		flagged := false
-		for _, allowedChar := range allowedChars {
-			if filenameChar == allowedChar {
-				flagged = true
-			}
-		}
-		// if not in allowed list replace by underscore
-		if !flagged {
-			cleanedName = strings.Replace(cleanedName, filenameChar, "_", 1)
-		}
-	}
-
-	return cleanedName
-}
-
-func virusScan(path string) error {
-	// will move file into quarantine directory if infected
-	cmd := exec.Command(
-		"/usr/bin/clamscan",
-		"--quiet",
-		"--infected",
-		"--recursive",
-		"--move="+quarantineDir,
-		"--max-scantime=600000", // 10 minutes
-		"--max-filesize=4000M",
-		"--max-scansize=4000M",
-		"--max-files=200",
-		"--max-recursion=6",
-		"--max-dir-recursion=6",
-		path,
-	)
-	err := cmd.Run()
-
-	return err
-}
-
-func main() {
-	var opts struct {
-		Scan bool `short:"s" long:"scan-virus" description:"Scan file for virus"`
-		Args struct {
-			SrcPaths []string `positional-arg-name:"path" required:"yes" description:"Paths of uploaded files"`
-		} `positional-args:"yes"`
-	}
-
-	if _, err := flags.Parse(&opts); err != nil {
-		os.Exit(1)
-	}
-
-	for _, srcPath := range opts.Args.SrcPaths {
-		// check that file is into incoming folder
-		if !strings.HasPrefix(srcPath, baseDir) {
-			log.Fatalln("file not in base dir (" + baseDir + "): " + srcPath)
-		}
-
-		// ensure permissions are correct
-		if err := setPermissions(srcPath); err != nil {
-			log.Fatalln(err)
-		}
-
-		// scan for virus if enabled
-		if opts.Scan {
-			if err := os.MkdirAll(quarantineDir, 0775); err != nil {
-				log.Fatalln(err)
-			}
-			if err := virusScan(srcPath); err != nil {
-				log.Fatalln(err)
-			}
-		}
-
-		// cleanup and set destination path
-		srcDir, srcFile := filepath.Split(srcPath)
-		dstFile := cleanName(srcFile)
-		dstDir := strings.ReplaceAll(srcDir, incomingDir, watchDir)
-		dstPath := dstDir + dstFile
-
-		// create destination directory
-		if err := os.MkdirAll(dstDir, 0775); err != nil {
-			log.Fatalln(err)
-		}
-
-		// move file into watchfolder
-		if err := os.Rename(srcPath, dstPath); err != nil {
-			log.Fatalln(err)
-		}
-
-		log.Println(srcPath + " moved to " + dstPath)
-	}
-}

roles/mediaimport/handlers/main.yml

@@ -29,8 +29,4 @@
     name: fail2ban
     state: restarted
 
-- name: sftp-verif
-  ansible.builtin.command:
-    cmd: timeout 30 sftp-verif
-
 ...

roles/mediaimport/tasks/main.yml

@@ -8,38 +8,17 @@
 
 ## USERS
 
-- name: create ftp folders
-  loop:
-    - /home/ftp/storage/incoming
-    - /home/ftp/storage/watchfolder
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    mode: '755'
-
-- name: deploy users management script
-  ansible.builtin.copy:
-    src: files/mediaimport.py
-    dest: /usr/local/bin/mediaimport
-    mode: '755'
-
 - name: create users
   loop: "{{ mediaimport_users }}"
   when:
     - item.name | d(false)
     - item.passwd | d(false)
   no_log: true
-  ansible.builtin.command: mediaimport add --yes --user {{ item.name }} --passwd {{ item.passwd }}
+  ansible.builtin.command: /usr/bin/mediaimportctl.py add --yes --user {{ item.name }} --passwd {{ item.passwd }}
   args:
     creates: /home/ftp/storage/incoming/{{ item.name }}
 
-- name: deploy on-upload script with setuid
-  ansible.builtin.copy:
-    src: files/on-upload
-    dest: /home/ftp/on-upload
-    mode: 04755
-
-## MYSECURESHELL
+## MEDIAIMPORT
 
 - name: enable password login for ssh
   notify: restart sshd
@@ -48,97 +27,6 @@
     regexp: "^PasswordAuthentication no"
     replace: "#PasswordAuthentication yes"
 
-- name: set the setuid on mysecureshell
-  ansible.builtin.file:
-    path: /usr/bin/mysecureshell
-    mode: 04755
-
-- name: configure mysecureshell
-  notify:
-    - restart mysecureshell
-    - sftp-verif
-  ansible.builtin.template:
-    src: sftp_config.j2
-    dest: /etc/ssh/sftp_config
-    mode: '644'
-
-## PURE-FTPD
-
-- name: set pure-ftpd default config
-  notify: restart pure-ftpd
-  ansible.builtin.copy:
-    dest: /etc/default/pure-ftpd-common
-    mode: '644'
-    content: |
-      STANDALONE_OR_INETD=standalone
-      VIRTUALCHROOT=false
-      UPLOADSCRIPT="/home/ftp/on-upload{% if mediaimport_virus_scan_on_upload %} --scan-virus{% endif %}"
-      UPLOADUID=0
-      UPLOADGID=0
-
-- name: configure pure-ftpd
-  notify: restart pure-ftpd
-  loop: "{{ mediaimport_pureftpd_config }}"
-  ansible.builtin.copy:
-    dest: /etc/pure-ftpd/conf/{{ item.key }}
-    content: "{{ item.value }}"
-    mode: '644'
-
-## PURE-FTPD CERTIFICATES
-
-- name: create certificate directory
-  ansible.builtin.file:
-    path: /etc/ssl/{{ ansible_fqdn }}
-    state: directory
-    mode: '755'
-
-- name: generate an private key
-  register: mediaimport_privkey
-  community.crypto.openssl_privatekey:
-    path: /etc/ssl/{{ ansible_fqdn }}/key.pem
-    mode: '600'
-
-- name: generate an csr
-  when: mediaimport_privkey is changed  # noqa no-handler
-  register: mediaimport_csr
-  openssl_csr:
-    path: /etc/ssl/{{ ansible_fqdn }}/csr.pem
-    privatekey_path: /etc/ssl/{{ ansible_fqdn }}/key.pem
-    common_name: "{{ ansible_fqdn }}"
-    mode: '600'
-
-- name: generate a self-signed certificate
-  when: mediaimport_csr is changed  # noqa no-handler
-  register: mediaimport_cert
-  openssl_certificate:
-    path: /etc/ssl/{{ ansible_fqdn }}/cert.pem
-    privatekey_path: /etc/ssl/{{ ansible_fqdn }}/key.pem
-    csr_path: /etc/ssl/{{ ansible_fqdn }}/csr.pem
-    provider: selfsigned
-    mode: '600'
-
-- name: concatenate key and certificate
-  when: mediaimport_cert is changed  # noqa no-handler
-  notify: restart pure-ftpd
-  ansible.builtin.shell: >
-    cat /etc/ssl/{{ ansible_fqdn }}/key.pem /etc/ssl/{{ ansible_fqdn }}/cert.pem > /etc/ssl/private/pure-ftpd.pem;
-    chmod 600 /etc/ssl/private/pure-ftpd.pem;
-
-- name: generate dhparams
-  notify: restart pure-ftpd
-  openssl_dhparam:
-    path: /etc/ssl/private/pure-ftpd-dhparams.pem
-    size: 1024
-    mode: '600'
-
-## MEDIAIMPORT
-
-- name: setup cron job
-  ansible.builtin.copy:
-    src: files/mediaimport
-    dest: /etc/cron.d/mediaimport
-    mode: '644'
-
 - name: configure mediaimport
   when:
     - mediaimport_ms_api_key | d(false)
@@ -155,6 +43,11 @@
     name: mediaimport
     enabled: true
 
+- name: enable mediaimport-cleanup timer
+  ansible.builtin.systemd:
+    name: mediaimport-cleanup.timer
+    enabled: true
+
 # FAIL2BAN
 
 - name: deploy fail2ban jail

roles/mediaimport/templates/sftp_config.j2

-## MySecureShell Configuration File
-# To get more informations on all possible options, please look at the doc:
-# http://mysecureshell.readthedocs.org
-
-#Default rules for everybody
-<Default>
-        GlobalDownload          50k
-        GlobalUpload            0
-        Download                5k
-        Upload                  0
-        StayAtHome              true
-        VirtualChroot           true
-        LimitConnection         100
-        LimitConnectionByUser   2
-        LimitConnectionByIP     10
-        Home                    /home/ftp/storage/incoming/$USER
-        CallbackUpload          "/home/ftp/on-upload{% if mediaimport_virus_scan_on_upload %} --scan-virus{% endif %} /home/ftp/storage/incoming/$USER$LAST_FILE_PATH"
-        IdleTimeOut             5m
-        ResolveIP               false
-        HideNoAccess            true
-        DefaultRights           0640 0750
-        ShowLinksAsLinks        false
-        LogFile                 /var/log/sftp-server.log
-        LogLevel                6
-        LogSyslog               true
-</Default>