Merge branch 't33830-security-repository-migration' into 'master'

Security repository migration | refs #33830

See merge request mediaserver/envsetup!70

.docker/Dockerfile

-FROM registry.ubicast.net/docker/debian-dev:latest
-
-# avoid warnings by switching to noninteractive
-ENV DEBIAN_FRONTEND noninteractive
-# local pyvenv to avoid conflicts with system
-ENV PYVENV ${HOME}/pyvenv
-# add pyvenv to path
-ENV PATH ${PYVENV}/bin:${PATH}
-
-# copy requirement file
-COPY ansible/requirements.dev.txt .
-
-RUN \
-    # install required tools
-    sudo apt-get update && \
-    sudo apt-get install -y libffi-dev libncurses5 libncursesw5 libssl-dev python3-netaddr && \
-    # create pyvenv + install ansible tools
-    python3 -m venv --system-site-packages ${PYVENV} && \
-    pip install -U pip wheel && \
-    pip install -r requirements.dev.txt && \
-    # clean up
-    sudo apt-get autoremove -y && \
-    sudo apt-get clean -y && \
-    sudo rm -rf /var/lib/apt/lists/* &&  \
-    sudo rm requirements.dev.txt 
-
-# switch back to dialog for any ad-hoc use of apt-get
-ENV DEBIAN_FRONTEND dialog
-ENV HOME /root
-USER root

.flake8

+# Run flake8 (pycodestyle + pyflakes) check.
+# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+# Ignored errors:
+# - E501: line too long
+# - E731: do not assign a lambda expression, use a def
+# - W503: line break before binary operator (deprecated rule)
+# - W505: doc line too long
+
+[flake8]
+
+ignore =
+    E501
+    E265
+    W503
+    W505
+
+exclude = .git,submodules

.githooks/pre-commit

-#!/bin/sh
-
-if git rev-parse --verify HEAD >/dev/null 2>&1; then
-        against=HEAD
-else
-        # Initial commit: diff against an empty tree object
-        against=$(git hash-object -t tree /dev/null)
-fi
-
-# Redirect output to stderr.
-exec 1>&2
-
-# Get changed files
-changed_files=$(git diff-index --name-only ${against})
-
-# Verify files content
-for file in ${changed_files}; do
-
-    if grep -qiE 'skyreach_(system|activation|api)_key' "${file}"; then
-    
-        # verify key
-        key=$(grep -iE 'skyreach_(system|activation|api)_key' "${file}" | grep -woiE '[a-z0-9]{32}')
-        if [ -n "${key}" ]; then
-            echo "Error: you are about to commit a secret key in file: ${file}"
-            echo "Please remove it before committing."
-            echo -
-            grep -iE 'skyreach_(system|activation|api)_key' "${file}" | grep -iE '[a-z0-9]{32}'
-            echo -
-            exit 1
-        fi
-    
-    fi
-
-done
-
-# vim:ft=sh

.gitignore

-# virtualenv
-.venv/
+*.py[cod]
 
-# python
-__pycache__/
-*.pyc
+# C extensions
+*.so
 
-# ansible
-ansible/inventories/_*
-ansible/inventories/local*/host_vars/localhost.yml
-ansible/inventories/offline*/host_vars/localhost.yml
-ansible/playbooks/_*
-ansible/roles/_*
-ansible/roles/elastic.elasticsearch
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+__pycache__
 
-# logs
-*.log
+# Installer logs
+pip-log.txt
 
-# packer
-packer_cache/
-output/
-packer/*.json
-!packer/example.json
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
 
-# ide
-.vscode/
-*.code-workspace
-.idea/
-*.sublime-workspace
-*.sublime-project
-
-# secrets
-.env/*
-!.env/_reset
-!.env/_config
-!.env/*example
-!.env/example
-
-# envsetup
-conf*.sh
-auto-generated-conf.sh*
-tests/scripts/ms-testing-suite
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject

.gitlab-ci.yml

----
-
-default:
-  image: registry.ubicast.net/mediaserver/envsetup
-
-stages:
-  - lint
-  - docker
-  - test-pf-std
-  - test-pf-ha
-  - test-pgsql
-
-# * * * * * * * * * * * * *
-
-lint:verify:
-  stage: lint
-  script:
-    - make lint
-
-# * * * * * * * * * * * * *
-
-# Docker envsetup image build job
-docker:image:
-  image: docker:stable
-  stage: docker
-  rules:
-    # Build docker image for schedule pipelines only
-    - if: '$DOCKER_BUILD == "True"'
+flake8:
+  image: python:3-alpine
+  tags:
+    - docker
   before_script:
-    - apk add bash make
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.ubicast.net
-  script:
-    - make docker-build
-    - make docker-push
-
-# * * * * * * * * * * * * *
-
-.test-template:
-  retry: 2
-  timeout: 4h
-  rules:
-    # Run deployment for schedule pipelines
-    - if: '$CI_PIPELINE_SOURCE == "schedule" && $PF_DEPLOY_TEST == "True"'
-      when: always
-    # Run deployment for manual pipelines
-    - if: '$CI_PIPELINE_SOURCE == "web"'
-      when: always
-    - when: never
-
-test:pf-std:
-  extends: .test-template
-  stage: test-pf-std
-  script:
-    - echo $CI_PIPELINE_SOURCE
-    - make test pf-std=1
-
-test:pf-ha:
-  extends: .test-template
-  stage: test-pf-ha
-  script:
-    - make test pf-ha=1
-
-test:pgsql-ha:
-  extends: .test-template
-  stage: test-pgsql
+    - python -m pip install --upgrade pip
+    - pip3 install flake8
   script:
-    - make test pgsql-ha=1 debug=1
-
-...
+    - flake8 .

.lint/ansible-apt-block-check.sh

-#!/usr/bin/env bash
-
-# config
-exclude_pattern=()
-exclude_pattern+=('^./ansible/roles/elastic.elasticsearch')
-
-apt_regex='^[^#]*apt:'
-until_regex='^[^#]*until: apt_status is success'
-
-# * * * 
-
-# go to repository root dir
-cd "$(readlink -f "$(dirname "${0}")")"/..
-
-# join function
-join_by() { local IFS="$1"; shift; echo "$*"; }
-
-# set all *.yml files to an array
-mapfile -t yml_files < <(find . -type f -iname '*.yml' | grep -vE "$(join_by '|' "${exclude_pattern[@]}")")
-
-# check every files
-errors_count=0
-for f in "${yml_files[@]}"; do
-
-    # count apt block
-    apt_block_count=$(grep -c "${apt_regex}" "${f}")
-
-    # test if file contain apt block
-    if (( apt_block_count > 0 )); then
-    
-        # get apt block, count apt: and until:
-        apt_blocks="$(awk -v RS='' "/${apt_regex}/" "${f}")"
-        apt_nb="$(echo "${apt_blocks}" | grep -c "${apt_regex}")"
-        until_nb="$(echo "${apt_blocks}" | grep -c "${until_regex}")"
-
-        # test if apt: and until: count differ
-        if (( apt_nb != until_nb )); then
-            echo "- ${f}"
-            (( errors_count++ ))
-        fi
-    fi
-
-done
-
-if (( errors_count != 0 )); then
-    echo "Files listed below contain incomplete apt blocks"
-    echo "Please refer to this documentation: https://docs.google.com/document/d/1B31l4v6VV_3r_ePPiugI8I_D_oRsUKFVIMIjerV_KvM/edit#heading=h.lm0b49ccpi46"
-    echo
-    exit 1
-else
-    exit 0
-fi
-

.lint/ansible-lint.conf

----
-
-exclude_paths:
-  - ansible/playbooks/_*.yml
-  - ansible/roles/_*/
-  - ansible/roles/elastic.elasticsearch/
-
-skip_list:
-  - '701'
-  - '403'
-  - '208' 
-  - '106' 
-
-...

.lint/flake8.conf

-[flake8]
-
-ignore =
-    E501
-    E265
-    W503
-    W505
-
-per-file-ignores =
-    ansible/roles/elastic.elasticsearch/*:E713

.lint/yamllint.conf

----
-
-extends: default
-
-ignore: |
-  .venv/
-  ansible/roles/elastic.elasticsearch/
-
-rules:
-  braces:
-    min-spaces-inside-empty: 0
-    max-spaces-inside-empty: 0
-    min-spaces-inside: 1
-    max-spaces-inside: 1
-    level: error
-  brackets:
-    min-spaces-inside-empty: 0
-    max-spaces-inside-empty: 0
-    min-spaces-inside: 1
-    max-spaces-inside: 1
-    level: error
-  truthy:
-    level: error
-  line-length: disable
-  indentation:
-    spaces: consistent
-    indent-sequences: true
-    check-multi-line-strings: false
-...

Makefile

-SHELL := /bin/bash
-DOCKER_IMAGE_NAME := registry.ubicast.net/mediaserver/envsetup
-VENV := /tmp/pyvensetup
-ANSIBLE_CONFIG := ansible/ansible.cfg
-PIP_BIN = $(shell command -v $(VENV)/bin/pip3 || command -v pip3 || echo pip3)
-PIP_COMPILE_BIN = $(shell command -v $(VENV)/bin/pip-compile || command -v pip-compile)
-ANSIBLE_BIN = $(shell command -v ansible || command -v $(VENV)/bin/ansible)
-ANSIBLE_PLAYBOOK_BIN = $(shell command -v ansible-playbook || command -v $(VENV)/bin/ansible-playbook)
-ANSIBLE_LINT_BIN = $(shell command -v ansible-lint || command -v $(VENV)/bin/ansible-lint)
-ANSIBLE_GALAXY_BIN = $(shell command -v ansible-galaxy || command -v $(VENV)/bin/ansible-galaxy || echo ansible-galaxy)
-YAMLLINT_BIN = $(shell command -v yamllint || command -v $(VENV)/bin/yamllint)
-FLAKE8_BIN = $(shell command -v flake8 || command -v $(VENV)/bin/flake8)
-
-# molecule tests flags
-ifdef debug
-	MOLECULE_FLAGS += --debug
-endif
-ifdef keep
-	MOLECULE_TEST_FLAGS += --destroy=never
-endif
-ifdef pf-std
-	MOLECULE_TEST_FLAGS += --scenario-name pf-std
-endif
-ifdef pf-ha
-	MOLECULE_TEST_FLAGS += --scenario-name pf-ha
-endif
-ifdef pgsql-ha
-	MOLECULE_TEST_FLAGS += --scenario-name pgsql-ha
-endif
-
-.PHONY: all
-## TARGET: DESCRIPTION: ARGS
-all: help
-
-.PHONY: venv
-## venv: Install python3-venv and create a temporary virtualenv
-venv:
-	-@command -v apt-get >/dev/null && apt-get update && apt-get install -y python3-venv
-	python3 -m venv $(VENV)
-
-## ansible/requirements.txt: Update requirements and their depende.dockeres
-## ansible/requirements.dev.txt: Update development requirements and their depende.dockeres
-%.txt: %.in
-	$(PIP_COMPILE_BIN) -U $^ -o $@
-	chmod 644 $@
-
-.PHONY: install
-## install: Install requirements
-install: venv
-	$(PIP_BIN) install -U pip wheel
-	$(PIP_BIN) install -r ansible/requirements.txt
-
-.PHONY: install-galaxy
-install-galaxy:
-	ANSIBLE_CONFIG=$(ANSIBLE_CONFIG) $(ANSIBLE_GALAXY_BIN) install -r ansible/requirements.yml
-
-.PHONY: install-dev
-## install-dev: Install development requirements
-install-dev: install
-	$(PIP_BIN) install -r ansible/requirements.dev.txt
-	[ -d .git/hooks ] || mkdir .git/hooks
-	ln -sfv .githooks/pre-commit .git/hooks/ || echo "Failed to create pre-commit link"
-
-.PHONY: lint
-## lint: Run linters on the project
-lint: 
-	$(FLAKE8_BIN) --config .lint/flake8.conf
-	$(YAMLLINT_BIN) --config-file .lint/yamllint.conf .
-	ANSIBLE_CONFIG=$(ANSIBLE_CONFIG) $(ANSIBLE_LINT_BIN) -c .lint/ansible-lint.conf ansible/playbooks/site.yml
-	.lint/ansible-apt-block-check.sh
-
-.PHONY: test
-## test: Run development tests on the project : SKYREACH_SYSTEM_KEY=<xxx>, debug=1, keep=1, pf-std=1, pgsql-ha=1
-test:
-	cd ansible; molecule $(MOLECULE_FLAGS) test $(MOLECULE_TEST_FLAGS)
-
-.PHONY: deploy
-## deploy: Run deployment playbooks : i=<inventory-path>, l=<host-or-group>, t=<tag>
-deploy:
-ifndef i
-	$(error i is undefined)
-endif
-ifndef l
-	$(eval l=all)
-endif
-ifndef t
-	$(eval t=all)
-endif
-	ANSIBLE_CONFIG=$(ANSIBLE_CONFIG) $(ANSIBLE_BIN) -i $(i) -l $(l) -m ping all
-	ANSIBLE_CONFIG=$(ANSIBLE_CONFIG) $(ANSIBLE_PLAYBOOK_BIN) -i $(i) ansible/playbooks/site.yml -e conf_update=true -l $(l) -t $(t)
-
-.PHONY: docker-build
-## docker-build: Run docker image build for.docker
-docker-build: docker-pull
-	docker build -t $(DOCKER_IMAGE_NAME) -f .docker/Dockerfile .
-
-.PHONY: docker-rebuild
-## docker-rebuild: Force docker image rebuild
-docker-rebuild:
-	docker build --pull --no-cache -t $(DOCKER_IMAGE_NAME) -f .docker/Dockerfile .
-
-.PHONY: docker-pull
-## docker-pull: Pull Docker image from registry
-docker-pull:
-	-docker pull $(DOCKER_IMAGE_NAME)
-
-.PHONY: docker-push
-## docker-push: Push Docker image to registry
-docker-push:
-	docker push $(DOCKER_IMAGE_NAME)
-
-.PHONY: help
-## help: Print this help message
-help:
-	@echo -e "Usage: \n"
-	@sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' | sed -e 's/^/ /'

README.md

-# EnvSetup
-
-## Usage
-
-To deploy UbiCast products:
-
-- [Install required tools](/doc/requirements.md)
-- [Configure hosts inventory](/doc/config.md)
-- [Deploy UbiCast softwares](/doc/deploy.md)
-
-To benchmark the solution:
-- [MediaServer Benchmark](/doc/bench.md)
-
-## Development
-
-To contribute:
-
-- [EnvSetup contributing guide](/doc/contrib.md)
-

ansible/ansible.cfg

-[defaults]
-
-# logging
-log_path = ansible.log
-
-# use python3 by default
-interpreter_python = /usr/bin/python3
-
-# disable output for skipped hosts and tasks
-display_skipped_hosts = false
-# skip ssh host key checking
-host_key_checking = false
-# disable creation of *.retry files when playbook fails
-retry_files_enabled = false
-
-# connect as root on hosts
-remote_user = root
-
-# cutom path for roles
-roles_path = roles
-# custom path for modules
-library = library
-# custom path for action plugins
-action_plugins = plugins/action
-
-# improve output format (with line return)
-stdout_callback = debug
-
-# ignore files directory
-inventory_ignore_patterns = files
-
-[ssh_connection]
-# enable pipelining to speed up ansible execution
-pipelining = True
-
-# add custom ssh options
-ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-
-# use scp instead of sftp
-scp_if_ssh = true

ansible/inventories/example-ha/group_vars/all.yml

----
-
-# customer name
-customer_short_name: customer
-
-# enable letsencrypt certificate
-letsencrypt_enabled: false
-
-# update conf.sh
-conf_update: false
-
-# repmgr configuration
-repmgr_password: my top secret repmgr default password
-repmgr_primary_node: "{{ hostvars['pg1']['ansible_default_ipv4']['address'] }}"
-
-# ha proxy configuration
-hap_config_listen:
-  - name: pgsql-primary
-    content: |2
-      bind localhost:54321
-      default-server inter 2s fall 3 rise 2 on-marked-down shutdown-sessions
-      option tcp-check
-      tcp-check expect string primary
-      maxconn 500
-      server pg1 192.168.122.1:5432 maxconn 500 check port 8543
-      server pg2 192.168.122.2:5432 maxconn 500 check port 8543 backup
-
-...

ansible/inventories/example-ha/group_vars/mediavault.yml

----
-
-# mediavault backup deployement
-# used by mediavault/add_backup.yml
-
-#mvt_base_dir: /backup
-#mvt_backups:
-#  - name: self-etc
-#    source: "/etc"
-#    dest: "{{ mvt_base_dir }}/self-etc"
-#  - name: data
-#    source: /data
-#    dest: "{{ mvt_base_dir }}/data"
-
-...

ansible/inventories/example-ha/host_vars/cs1.yml

----
-
-skyreach_system_key: changeme
-
-...

ansible/inventories/example-ha/host_vars/mi1.yml

----
-
-skyreach_system_key: changeme
-
-...

ansible/inventories/example-ha/host_vars/mm1.yml

----
-
-skyreach_system_key: changeme
-
-...

ansible/inventories/example-ha/host_vars/mo1.yml

----
-
-skyreach_system_key: changeme
-
-...

ansible/inventories/example-ha/host_vars/ms1.yml

----
-
-skyreach_system_key: changeme
-
-...

ansible/inventories/example-ha/host_vars/ms2.yml

----
-
-skyreach_system_key: changeme
-
-...