Merge branch 't30039-reliability' into 'master'

make tests more reliable | refs #30039

See merge request mediaserver/envsetup!1

.devcontainer/Dockerfile

-FROM ubuntu:bionic
+FROM registry.ubicast.net/docker/debian-dev:buster
 
-# Set the default shell to bash instead of sh
-ENV SHELL /bin/bash
-# Update path to include virtualenv
-ENV PATH "/usr/local/pyvenv/bin:${PATH}"
-# Set locales
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-# Configure apt
+# avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends apt-utils 2>&1
+# local pyvenv to avoid conflicts with system
+ENV PYVENV ${HOME}/pyvenv
+# add pyvenv to path
+ENV PATH ${PYVENV}/bin:${PATH}
 
-# Install git, required tools
-RUN apt-get update && \
-    apt-get install -y \
-        apt-transport-https \
-        bash-completion \
-        build-essential \
-        ca-certificates \
-        curl \
-        git \
-        gnupg-agent \
+RUN \
+    # install required tools
+    sudo apt-get update && \
+    sudo apt-get install -y \
         libffi-dev \
         libssl-dev \
-        lsb-release \
-        procps \
-        python3 \
-        python3-dev \
-        python3-venv \
-        shellcheck \
-        software-properties-common \
-        unzip \
-        vim-tiny \
-    2>&1
-
-# Configure shell
-RUN sed -i 's/#force_color_prompt=yes/force_color_prompt=yes/' /root/.bashrc && \
-    echo ". /etc/bash_completion" >> /root/.bashrc
-
-# Install Docker CE CLI.
-RUN curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | apt-key add - 2>/dev/null && \
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" && \
-    apt-get update && \
-    apt-get install -y docker-ce-cli
+        && \
+    # clean up
+    sudo apt-get autoremove -y && \
+    sudo apt-get clean -y && \
+    sudo rm -rf /var/lib/apt/lists/* &&  \
+    # create pyvenv
+    python3 -m venv ${PYVENV} && \
+    # update pip tools into pyvenv
+    pip install -U \
+        pip \
+        wheel \
+    && \
+    :
 
-# Install Poetry
-RUN python3 -m venv /usr/local/pyvenv && \
-    pip install -U pip wheel 2>&1 && \
-    pip install psutil pyOpenSSL pyspf requests defusedxml 2>&1 && \
-    pip install black flake8 pre-commit pylint pysnooper 2>&1
+RUN \
+    # requirements
+    pip install psutil pyOpenSSL pyspf requests defusedxml && \
+    # dev requirements
+    pip install black flake8 pre-commit pylint pysnooper && \
+    :
 
-# Clean up
-RUN apt-get autoremove -y && \
-    apt-get clean -y && \
-    rm -rf /var/lib/apt/lists/*
+# switch back to dialog for any ad-hoc use of apt-get
 ENV DEBIAN_FRONTEND=dialog

.devcontainer/devcontainer.json

 {
-    "name": "envsetup",
     "dockerComposeFile": "docker-compose.yml",
     "service": "app",
     "workspaceFolder": "/workspace",
+    "postCreateCommand": "bash .devcontainer/scripts/postcreate.sh",
     "extensions": [
-        // editor
-        "editorconfig.editorconfig",
-        "mikestead.dotenv",
-        // python
-        "ms-python.python",
-        // docker
-        "peterjausovec.vscode-docker"
-    ]
+      // python
+      "ms-python.python",
+    ],
+    "settings": {
+    },
 }

.devcontainer/docker-compose.yml

 ---
 
-version: '3'
+version: "3"
 
 services:
   app:
     build:
-      context: ..
-      dockerfile: .devcontainer/Dockerfile
+      context: ".."
+      dockerfile: ".devcontainer/Dockerfile"
+    user: "vscode"
     volumes:
-      - ..:/workspace
-      - ~/.config/git/config:/root/.config/git/config
-      - ~/.config/git/ignore:/root/.config/git/ignore
-      - ${SSH_AUTH_SOCK}:/ssh-agent
-      - /var/run/docker.sock:/var/run/docker.sock
+      - "~/.config/git:/home/vscode/.config/git:ro"
+      - "~/.ssh:/home/vscode/.ssh:ro"
+      - "${SSH_AUTH_SOCK}:/ssh-agent:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "..:/workspace"
     environment:
-      - SSH_AUTH_SOCK=/ssh-agent
-    working_dir: /workspace
-    command: sleep infinity
+      - "SSH_AUTH_SOCK=/ssh-agent"
+    working_dir: "/workspace"
+    command: "sleep infinity"
 
 ...

.devcontainer/scripts/postcreate.sh

+#!/usr/bin/env bash
+
+# make the GID match with docker on the host
+sudo groupmod -g $(ls -ln /var/run/docker.sock | awk '{ print $4 }') docker

.flake8

-[flake8]
-# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
-# Ignored errors:
-# - E501: line too long
-# - W503: line break before binary operator (deprecated rule)
-# - W505: doc line too long
-ignore=E501,W503,W505

.pre-commit-config.yaml

----
-
-repos:
-  - repo: https://github.com/ambv/black
-    rev: 19.3b0
-    hooks:
-      - id: black
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 45fc394c19e208123b8e9ec3f584c7ae3adef8c4
-    hooks:
-      - id: flake8
-
-...

setup.cfg

+[metadata]
+name = envsetup
+long_description = file: README.md
+long_description_content_type = text/markdown
+
+[options]
+setup_requires=
+    setuptools
+    wheel
+
+[options.extras_require]
+dev =
+    black
+    flake8
+    pylint
+    pysnooper
+
+[flake8]
+exclude =
+    .venv/
+ignore =
+    E501
+    W503
+    W505
+max-line-length = 88
+
+[pylint]
+ignore=
+disable=
+    bad-continuation,
+    invalid-name,
+    missing-docstring,
+    too-few-public-methods,
+    too-many-locals,

setup.py

+#!/usr/bin/env python3
+
+import setuptools
+
+
+setuptools.setup()

tester.py

@@ -13,6 +13,7 @@ import subprocess
 import sys
 import uuid
 import glob
+from time import sleep
 
 import utils
 from utils import log
@@ -324,13 +325,20 @@ class Tester:
             start_date = datetime.datetime.utcnow()
             log("Test start: %s UTC." % start_date.strftime("%Y-%m-%d %H:%M:%S"))
             # Run test
-            p = subprocess.Popen(
-                command,
-                stdin=sys.stdin,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-            )
-            out, err = p.communicate()
+            count = 0
+            while count < 3:
+                count += 1
+                log("Attempt: %s" % str(count))
+                p = subprocess.Popen(
+                    command,
+                    stdin=sys.stdin,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                )
+                out, err = p.communicate()
+                if p.returncode in (0, 2, 3):
+                    break
+                sleep(5 * count * count)
             if out:
                 out = out.decode("utf-8").strip()
                 out_of_support = out_of_support or "out of support" in out