Do not update Nginx vhosts files in Nginx setup (refs #27623).

2.Common_services/5.Nginx/0_setup.py

@@ -6,24 +6,7 @@ import re
 import utils
 
 
-def vhost_write_cmds(dir_path, name, server_name, **data):
-    params = [('{{ server_name }}', server_name)]
-    for k, v in data.items():
-        params.append(('{{ %s }}' % k, v))
-    return [
-        dict(
-            line='write',
-            template='%s/vhost_%s.conf' % (dir_path, name),
-            target='/etc/nginx/sites-available/%s.conf' % name,
-            params=params,
-        ),
-        'ln -sfn ../sites-available/%s.conf /etc/nginx/sites-enabled/%s.conf' % (name, name),
-        'echo "Vhost %s updated (domain: %s)."' % (name, server_name),
-    ]
-
-
 def setup(interactive=True):
-    dir_path = utils.get_dir(__file__)
     cmds = [
         'apt-get remove -y apache2',
         'apt-get install -y nginx',
@@ -32,77 +15,25 @@ def setup(interactive=True):
     ]
     hosts = list()
     need_uwsgi = False
-    # FTP vhost (deprecated)
-    if os.path.exists('/var/www/videos'):
-        cmds.append('cp %s/crossdomain.xml /var/www/videos/crossdomain.xml' % dir_path)
-        server_name = utils.get_conf('FTP_SERVER_NAME') or 'videos'
-        cmds.extend(vhost_write_cmds(dir_path, 'videos', server_name))
-        hosts.append(server_name)
-    # Wowza vhost (deprecated)
-    if os.path.exists('/var/www/streaming'):
-        cmds.append('cp %s/crossdomain.xml /var/www/streaming/crossdomain.xml' % dir_path)
-        server_name = utils.get_conf('WOWZA_SERVER_NAME') or 'streaming'
-        cmds.extend(vhost_write_cmds(dir_path, 'streaming', server_name))
-        hosts.append(server_name)
     # MediaServer vhost (mediaserver-msuser)
     if os.path.exists('/home/msuser/msinstance'):
         need_uwsgi = True
-        server_name = utils.get_conf('MS_SERVER_NAME') or 'mediaserver'
-        worker_ips = utils.get_conf('CELERITY_WORKER_IP') or ''
-        whitelist = ''
-        for worker_ip in worker_ips.split(','):
-            worker_ip = worker_ip.strip()
-            if worker_ip and worker_ip != '127.0.0.1':
-                whitelist += '\n\t%s 1;' % worker_ip
-        cmds.extend(vhost_write_cmds(dir_path, 'mediaserver-msuser', server_name,
-            secret=utils.get_conf('MS_SECRET') or 'secret', whitelist=whitelist))
-        hosts.append(server_name)
+        hosts.append(utils.get_conf('MS_SERVER_NAME') or 'mediaserver')
     # Monitor vhost
     if os.path.exists('/home/msmonitor/msmonitor'):
         need_uwsgi = True
-        server_name = utils.get_conf('MONITOR_SERVER_NAME') or 'msmonitor'
-        cmds.extend(vhost_write_cmds(dir_path, 'msmonitor', server_name))
-        hosts.append(server_name)
+        hosts.append(utils.get_conf('MONITOR_SERVER_NAME') or 'msmonitor')
     # SkyReach vhost
     if os.path.exists('/home/skyreach/htdocs'):
         need_uwsgi = True
-        server_name = utils.get_conf('CM_SERVER_NAME') or 'campusmanager'
-        cmds.extend(vhost_write_cmds(dir_path, 'skyreach', server_name))
-        hosts.append(server_name)
-    # Cache vhost
-    if os.path.exists('/var/www/cache'):
-        cmds.append('cp %s/crossdomain.xml /var/www/cache/crossdomain.xml' % dir_path)
-        server_name = utils.get_conf('CACHE_SERVER_NAME') or 'cache'
-        cmds.extend(vhost_write_cmds(dir_path, 'cache', server_name, source_server=utils.get_conf('CACHE_SOURCE') or 'http://undefined'))
-        hosts.append(server_name)
+        hosts.append(utils.get_conf('CM_SERVER_NAME') or 'campusmanager')
+    # TODO: get domains by parsing files in sites-enabled
     if need_uwsgi:
         cmds.append('apt-get install -y uwsgi uwsgi-plugin-python3')
     utils.run_commands(cmds)
     # Update hosts file
-    rc, hostname = utils.exec_cmd('hostname')
-    if rc == 0 and hostname not in hosts:
-        hosts.insert(0, hostname)
-    with open('/etc/hosts', 'r') as fo:
-        content = fo.read()
-    new_content = list()
-    found_127 = False
-    for line in content.split('\n'):
-        if not found_127 and line.startswith('127.0.0.1'):
-            found_127 = True
-            for host in hosts:
-                if ' ' + host not in line:
-                    line += ' ' + host
-                    utils.log('Adding host %s to /etc/hosts 127.0.0.1 aliases.' % host)
-        new_content.append(line)
-    if not found_127:
-        new_content.append('127.0.0.1 %s' % ' '.join(hosts))
-    new_content = '\n'.join(new_content)
-    if new_content != content:
-        with open('/etc/hosts', 'w') as fo:
-            fo.write(new_content)
-        utils.log('/etc/hosts updated.')
-    else:
-        utils.log('/etc/hosts is already up to date.')
+    if hosts:
+        utils.add_hosts_to_localhost(hosts)
     # Move ssl.conf
     ssl_conf = '/etc/nginx/conf.d/ssl.conf'
     if os.path.exists(ssl_conf):

2.Common_services/5.Nginx/vhost_mediaserver-msuser.conf

-geo $msuser_whitelist {
-	default 0;
-	127.0.0.1 1;{{ whitelist }}
-}
-map $msuser_whitelist $msuser_allowed {
-	0 $secure_link;
-	1 1;
-}
-
-server {
-	listen 80;
-	server_name {{ server_name }};
-
-	location /.well-known/acme-challenge {
-		default_type "text/plain";
-		root /tmp/letsencrypt;
-	}
-	location / {
-		return 301 https://$host$request_uri;
-	}
-}
-
-server {
-	listen 443 ssl http2;
-	server_name {{ server_name }};
-	root /var/www/msuser;
-
-	access_log /var/log/nginx/access_msuser.log;
-	error_log /var/log/nginx/error_msuser.log;
-
-	location /static {
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-	location /public {
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-	location /crossdomain {
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-	location /robots {
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-	location /resources {
-		location ~ \.ts$ {
-		}
-		location ~ \.xml$ {
-		}
-		secure_link $arg_st,$arg_e;
-		secure_link_md5 "{{ secret }}$uri$arg_e$remote_addr";
-		# link expired
-		if ($msuser_allowed = "0") {
-			return 410;
-		}
-		# bad hash
-		if ($msuser_allowed = "") {
-			return 403;
-		}
-	}
-	location /streaming/ {
-		rewrite ^/streaming/(.*)$ /live/$1? break;
-		proxy_pass http://localhost:1935;
-		proxy_cache livecache;
-		# do not consider secure urls as new files
-		proxy_cache_key $scheme$proxy_host$uri;
-		# only one request at a time will be allowed to populate a new cache element
-		proxy_cache_lock on;
-		# show if playlist or fragment has been cached from wowza
-		add_header X-Cache $upstream_cache_status;
-		# wowza is defining expiration headers (m3u8 is 1s, ts is 3600)
-	}
-	location / {
-		uwsgi_pass unix:///home/msuser/mstmp/uwsgi.sock;
-		include /etc/nginx/uwsgi_params;
-	}
-}

2.Common_services/5.Nginx/vhost_msmonitor.conf

-server {
-	listen 80;
-	server_name {{ server_name }};
-
-	location /.well-known/acme-challenge {
-		default_type "text/plain";
-		root /tmp/letsencrypt;
-	}
-	location / {
-		return 301 https://$host$request_uri;
-	}
-}
-
-server {
-	listen 443 ssl;
-	server_name {{ server_name }};
-	root /var/www/msmonitor;
-
-	access_log /var/log/nginx/access_msmonitor.log;
-	error_log /var/log/nginx/error_msmonitor.log;
-
-	location /media {
-	}
-	location /static {
-	}
-	location /robots {
-	}
-	location /wowza-manager/ {
-		proxy_pass http://localhost:8088/;
-		proxy_set_header X-Forwarded-Host $host;
-		proxy_set_header X-Forwarded-Server $host;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-		proxy_cookie_path / /wowza-manager/;
-	}
-	location / {
-		uwsgi_pass unix:///home/msmonitor/msmonitor/temporary/uwsgi.sock;
-		include /etc/nginx/uwsgi_params;
-	}
-}

2.Common_services/5.Nginx/vhost_skyreach.conf

-map $http_upgrade $connection_upgrade {
-	default upgrade;
-	'' close;
-}
-
-server {
-	listen 80;
-	server_name {{ server_name }};
-
-	location /.well-known/acme-challenge {
-		default_type "text/plain";
-		root /tmp/letsencrypt;
-	}
-	location / {
-		return 301 https://$host$request_uri;
-	}
-}
-
-server {
-	listen 443 ssl;
-	server_name {{ server_name }};
-	root /home/skyreach/htdocs/skyreach_site/static;
-
-	access_log /var/log/nginx/access_skyreach.log;
-	error_log /var/log/nginx/error_skyreach.log;
-
-	location /media {
-		alias /home/skyreach/htdocs/skyreach_site/media;
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-	location /static {
-		alias /home/skyreach/htdocs/skyreach_site/static;
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-	location /robots.txt {
-		alias /home/skyreach/htdocs/skyreach_site/static/robots.txt;
-		expires 30d;
-		add_header Pragma public;
-		add_header Cache-Control "public";
-	}
-
-	location /remote-event/ {
-		proxy_pass http://127.0.0.1:6586/;
-		proxy_set_header X-Real-IP $remote_addr;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-		proxy_read_timeout 12h;
-	}
-
-	location /proxy/ {
-		# proxy to local SSH tunnels (for MediaCoder UI and files)
-		# ?: means that the group should not be stored as variable
-
-		# 404 for ports outside of range (40000-60000)
-		location ~ "^/proxy/(?!(4|5\d{4}/)).*$" {
-			return 404;
-		}
-
-		# proxy for m3u8 and ts files with cache
-		location ~ "^/proxy/(\d+)/(.+\.(?:m3u8|ts))$" {
-			proxy_cache livecache;
-			# do not consider secure urls as new files
-			proxy_cache_key $scheme$proxy_host$uri;
-			# only one request at a time will be allowed to populate a new cache element
-			proxy_cache_lock on;
-			# show if playlist or fragment has been cached from wowza
-			add_header X-Cache $upstream_cache_status;
-			# wowza is defining expiration headers (m3u8 is 1s, ts is 3600)
-			proxy_pass https://127.0.0.1:$1/$2;
-		}
-
-		# proxy for other urls including websocket
-		location ~ "^/proxy/(\d+)/(.*)$" {
-			proxy_http_version 1.1;
-			proxy_buffering off;
-			# needed for websocket
-			proxy_set_header Upgrade $http_upgrade;
-			proxy_set_header Connection $connection_upgrade;
-			# change scheme of "Origin" to https
-			proxy_set_header Origin https://127.0.0.1:$1;
-			proxy_set_header X-Real-IP $remote_addr;
-			proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header X-Forwarded-Proto $scheme;
-			proxy_pass https://127.0.0.1:$1/$2$is_args$args;
-		}
-	}
-
-	location / {
-		uwsgi_pass unix:///home/skyreach/.skyreach/uwsgi.sock;
-		include /etc/nginx/uwsgi_params;
-	}
-
-	# APT proxy urls
-	location ~ "/[\w\d\.\-\_]+.ubuntu.com/ubuntu" {
-		proxy_pass http://127.0.0.1:3142;
-	}
-}

30.Deprecated/2.FTP/0_setup.py

@@ -27,8 +27,6 @@ def setup(interactive=True):
         'chmod -R 777 /home/ftp/storage/msuploads',
         'chmod -R 775 /home/ftp/storage/www',
         'chown -R ftp:ftp /home/ftp/storage',
-        'mkdir -p /var/www',
-        'ln -sfn /home/ftp/storage/www /var/www/videos',
         # Config
         'echo "no" > /etc/pure-ftpd/conf/AllowDotFiles',
         'echo "yes" > /etc/pure-ftpd/conf/CallUploadScript',
@@ -67,3 +65,18 @@ def setup(interactive=True):
         # Restore password conf if required
         if os.path.exists('pureftpd.passwd.tmp'):
             os.rename('pureftpd.passwd.tmp', pwd_path)
+
+    # Web server onfiguration
+    server_name = utils.get_conf('FTP_SERVER_NAME') or 'videos'
+    cmds = [
+        'apt-get remove -y apache2',
+        'apt-get install -y nginx',
+        'mkdir -p /var/www',
+        'ln -sfn /home/ftp/storage/www /var/www/videos',
+        'cp %s/crossdomain.xml /home/ftp/storage/www/crossdomain.xml' % dir_path,
+        'cp %s/vhost_videos.conf /etc/nginx/sites-available/videos.conf' % dir_path,
+        'sed -i "s@server_name videos;@server_name %s;@" "/etc/nginx/sites-available/videos.conf"' % server_name,
+        'ln -sfn ../sites-available/videos.conf /etc/nginx/sites-enabled/videos.conf',
+    ]
+    utils.run_commands(cmds)
+    utils.add_hosts_to_localhost([server_name])

2.Common_services/5.Nginx/vhost_videos.conf → 30.Deprecated/2.FTP/vhost_videos.conf

 server {
 	listen 80;
 	listen 443 ssl;
-	server_name {{ server_name }};
+	server_name videos;
 
 	root /var/www/videos/;
 

8.MediaCache/1.Install_cache/0_setup.py

@@ -5,10 +5,19 @@ import utils
 
 def setup(interactive=True):
     dir_path = utils.get_dir(__file__)
+    server_name = utils.get_conf('CACHE_SERVER_NAME') or 'cache'
+    source_server = utils.get_conf('CACHE_SOURCE') or 'http://undefined'
     cmds = [
+        'apt-get remove -y apache2',
+        'apt-get install -y nginx',
         'mkdir -p /var/www/cache',
         'cp %s/index.html /var/www/cache/index.html' % dir_path,
         'cp %s/crossdomain.xml /var/www/cache/crossdomain.xml' % dir_path,
         'cp %s/nginx-limits.conf /etc/security/limits.d/nginx.conf' % dir_path,
+        'cp %s/vhost_cache.conf /etc/nginx/sites-available/cache.conf' % dir_path,
+        'sed -i "s@server_name cache;@server_name %s;@" "/etc/nginx/sites-available/cache.conf"' % server_name,
+        'sed -i "s@proxy_pass http://undefined;@proxy_pass %s;@" "/etc/nginx/sites-available/cache.conf"' % source_server,
+        'ln -sfn ../sites-available/cache.conf /etc/nginx/sites-enabled/cache.conf',
     ]
     utils.run_commands(cmds)
+    utils.add_hosts_to_localhost([server_name])

2.Common_services/5.Nginx/vhost_cache.conf → 8.MediaCache/1.Install_cache/vhost_cache.conf

@@ -8,7 +8,7 @@ log_format cache '$remote_addr - $host [$time_local] "$request" $status '
 server {
 	listen 80 default_server;
 	listen 443 default_server ssl;
-	server_name {{ server_name }};
+	server_name cache;
 
 	root /var/www/cache/;
 
@@ -26,7 +26,7 @@ server {
 	location /streaming/ {
 		# Live; expiration headers are defined by upstream (nginx/wowza)
 		rewrite ^/(.*)$ /$1? break;
-		proxy_pass {{ source_server }};
+		proxy_pass http://undefined;
 		proxy_cache uc-cache;
 		# do not consider secure urls as new files
 		proxy_cache_key $scheme$proxy_host$uri;
@@ -41,7 +41,7 @@ server {
 		# VOD
 		location ~ \.(m3u8|ts|mp4|mp3|webm|oga|ogv|ogg|mov|flv)$ {
 			rewrite ^/(.*)$ /$1? break;
-			proxy_pass {{ source_server }};
+			proxy_pass http://undefined;
 			proxy_cache uc-cache;
 			# do not consider secure urls as new files
 			proxy_cache_key $scheme$proxy_host$uri;

utils.py

@@ -238,9 +238,9 @@ def run_commands(cmds: list):
                 if not cmd.get("target"):
                     raise Exception("No target file to write in.")
                 if (
-                    cmd.get("backup")
-                    and os.path.exists(cmd["target"])
-                    and not os.path.exists(cmd["target"] + ".back")
+                    cmd.get("backup") and
+                    os.path.exists(cmd["target"]) and
+                    not os.path.exists(cmd["target"] + ".back")
                 ):
                     os.rename(cmd["target"], cmd["target"] + ".back")
                     log("A backup file has been created for:\n%s" % cmd["target"])
@@ -276,3 +276,37 @@ def run_commands(cmds: list):
     except Exception as e:
         log("Command failed:\n%s" % e)
         raise
+
+
+def add_hosts_to_localhost(hosts: list):
+    """Add a list of hosts to 127.0.0.1 in /etc/hosts.
+
+    :param hosts: List of commands
+    :type hosts: list
+    :raises Exception: Houston we have a problem
+    """
+
+    rc, hostname = exec_cmd('hostname')
+    if rc == 0 and hostname not in hosts:
+        hosts.insert(0, hostname)
+    with open('/etc/hosts', 'r') as fo:
+        content = fo.read()
+    new_content = list()
+    found_127 = False
+    for line in content.split('\n'):
+        if not found_127 and line.startswith('127.0.0.1'):
+            found_127 = True
+            for host in hosts:
+                if ' ' + host not in line:
+                    line += ' ' + host
+                    log('Adding host %s to /etc/hosts 127.0.0.1 aliases.' % host)
+        new_content.append(line)
+    if not found_127:
+        new_content.append('127.0.0.1 %s' % ' '.join(hosts))
+    new_content = '\n'.join(new_content)
+    if new_content != content:
+        with open('/etc/hosts', 'w') as fo:
+            fo.write(new_content)
+        log('/etc/hosts updated.')
+    else:
+        log('/etc/hosts is already up to date.')