From a8204415773e3149eb43c227fc4ea63f4b434bcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Diemer?= <stephane.diemer@ubicast.eu>
Date: Wed, 8 Feb 2017 11:28:49 +0100
Subject: [PATCH] Changed cache setup and cache conf (refs #20384).
---
2.Common_services/5.Nginx/0_setup.py | 26 ++-
2.Common_services/5.Nginx/vhost_cache.conf | 92 +++++++++
8.Cache/1.Install_cache/0_setup.py | 20 ++
8.Cache/1.Install_cache/0_setup.sh | 25 ---
8.Cache/1.Install_cache/default | 175 ------------------
8.Cache/1.Install_cache/index.html | 32 ++--
.../2.Deploy_munin_configuration/0_setup.sh | 5 -
.../0_setup.sh | 0
.../ferm.conf | 0
launcher.sh | 3 +-
10 files changed, 152 insertions(+), 226 deletions(-)
create mode 100644 2.Common_services/5.Nginx/vhost_cache.conf
create mode 100644 8.Cache/1.Install_cache/0_setup.py
delete mode 100755 8.Cache/1.Install_cache/0_setup.sh
delete mode 100644 8.Cache/1.Install_cache/default
delete mode 100755 8.Cache/2.Deploy_munin_configuration/0_setup.sh
rename 8.Cache/{3.Install_ferm => 2.Install_ferm}/0_setup.sh (100%)
rename 8.Cache/{3.Install_ferm => 2.Install_ferm}/ferm.conf (100%)
diff --git a/2.Common_services/5.Nginx/0_setup.py b/2.Common_services/5.Nginx/0_setup.py
index 228843ff..bc9725dc 100644
--- a/2.Common_services/5.Nginx/0_setup.py
+++ b/2.Common_services/5.Nginx/0_setup.py
@@ -25,38 +25,52 @@ def vhost_write_cmds(dir_path, name, server_name, **data):
def setup(interactive=True):
dir_path = utils.get_dir(__file__)
cmds = [
- 'apt-get install --yes nginx uwsgi uwsgi-plugin-python3',
+ 'apt-get remove -y apache2'
+ 'apt-get install -y nginx',
'rm -f /etc/nginx/sites-enabled/default',
+ 'rm -f /etc/nginx/sites-enabled/default.conf',
]
hosts = list()
- # FTP: videos vhost
+ need_uwsgi = False
+ # FTP vhost (deprecated)
if os.path.exists('/var/www/videos'):
cmds.append('cp %s/crossdomain.xml /var/www/videos/crossdomain.xml' % dir_path)
server_name = utils.get_conf('FTP_SERVER_NAME') or 'videos'
cmds.extend(vhost_write_cmds(dir_path, 'videos', server_name))
hosts.append(server_name)
- # Wowza: streaming vhost
+ # Wowza vhost (deprecated)
if os.path.exists('/var/www/streaming'):
cmds.append('cp %s/crossdomain.xml /var/www/streaming/crossdomain.xml' % dir_path)
server_name = utils.get_conf('WOWZA_SERVER_NAME') or 'streaming'
cmds.extend(vhost_write_cmds(dir_path, 'streaming', server_name))
hosts.append(server_name)
- # MediaServer: mediaserver-msuser vhost
+ # MediaServer vhost (mediaserver-msuser)
if os.path.exists('/home/msuser/msinstance'):
+ need_uwsgi = True
server_name = utils.get_conf('MS_SERVER_NAME') or 'mediaserver'
cmds.extend(vhost_write_cmds(dir_path, 'mediaserver-msuser', server_name,
secret=utils.get_conf('MS_SECRET') or 'secret', worker_ip=utils.get_conf('CELERITY_WORKER_IP') or '127.0.1.1'))
hosts.append(server_name)
- # Monitor: msmonitor vhost
+ # Monitor vhost
if os.path.exists('/home/msmonitor/msmonitor'):
+ need_uwsgi = True
server_name = utils.get_conf('MONITOR_SERVER_NAME') or 'msmonitor'
cmds.extend(vhost_write_cmds(dir_path, 'msmonitor', server_name))
hosts.append(server_name)
- # SkyReach: skyreach vhost
+ # SkyReach vhost
if os.path.exists('/home/skyreach/htdocs'):
+ need_uwsgi = True
server_name = utils.get_conf('CM_SERVER_NAME') or 'skyreach'
cmds.extend(vhost_write_cmds(dir_path, 'skyreach', server_name))
hosts.append(server_name)
+ # Cache vhost
+ if os.path.exists('/var/www/cache'):
+ cmds.append('cp %s/crossdomain.xml /var/www/cache/crossdomain.xml' % dir_path)
+ server_name = utils.get_conf('CACHE_SERVER_NAME') or 'cache'
+ cmds.extend(vhost_write_cmds(dir_path, 'cache', server_name, source_server=utils.get_conf('CACHE_SOURCE') or 'http://undefined'))
+ hosts.append(server_name)
+ if need_uwsgi:
+ cmds.append('apt-get install -y uwsgi uwsgi-plugin-python3')
utils.run_commands(cmds)
# Update hosts file
rc, hostname = utils.exec_cmd('hostname')
diff --git a/2.Common_services/5.Nginx/vhost_cache.conf b/2.Common_services/5.Nginx/vhost_cache.conf
new file mode 100644
index 00000000..0339e19a
--- /dev/null
+++ b/2.Common_services/5.Nginx/vhost_cache.conf
@@ -0,0 +1,92 @@
+proxy_cache_path /tmp/nginx-uc-cache levels=1:2 keys_zone=uc-cache:10m max_size=10g inactive=300s;
+
+server {
+ listen 80 default_server;
+ listen 443 default_server ssl;
+ server_name {{ server_name }};
+
+ root /var/www/cache/;
+
+ access_log /var/log/nginx/access_cache.log;
+ error_log /var/log/nginx/error_cache.log;
+
+ location /crossdomain {
+ }
+
+ location /streaming/ {
+ # Live
+ location ~ \.m3u8$ {
+ rewrite ^/(.*)$ /$1? break;
+ proxy_pass {{ source_server }};
+ proxy_cache uc-cache;
+ # do not consider secure urls as new files
+ proxy_cache_key $scheme$proxy_host$uri;
+ # only one request at a time will be allowed to populate a new cache element
+ proxy_cache_lock on;
+ proxy_cache_min_uses 1;
+ proxy_cache_use_stale updating;
+ # how long should the data be kept in the cache
+ proxy_cache_valid 200 1s;
+ # instruct browser never to cache this
+ expires -1;
+ # headers
+ proxy_ignore_headers Cache-Control;
+ proxy_ignore_headers Set-Cookie;
+ proxy_ignore_headers X-Accel-Expires;
+ proxy_ignore_headers Expires;
+ proxy_hide_header Pragma;
+ add_header X-Cache $upstream_cache_status;
+ }
+ location ~ \.ts$ {
+ rewrite ^/(.*)$ /$1? break;
+ proxy_pass {{ source_server }};
+ proxy_cache uc-cache;
+ # do not consider secure urls as new files
+ proxy_cache_key $scheme$proxy_host$uri;
+ # only one request at a time will be allowed to populate a new cache element
+ proxy_cache_lock on;
+ proxy_cache_min_uses 1;
+ proxy_cache_use_stale updating;
+ # how long should the data be kept in the cache
+ proxy_cache_valid 200 10s;
+ # instruct browser to cache this
+ expires 2s;
+ # headers
+ proxy_ignore_headers Cache-Control;
+ proxy_ignore_headers Set-Cookie;
+ proxy_ignore_headers X-Accel-Expires;
+ proxy_ignore_headers Expires;
+ proxy_hide_header Pragma;
+ add_header X-Cache $upstream_cache_status;
+ }
+ }
+ location /resources/ {
+ # VOD
+ location ~ \.(m3u8|ts|mp4|mp3|oga|ogv|ogg|mov|flv)$ {
+ rewrite ^/(.*)$ /$1? break;
+ proxy_pass {{ source_server }};
+ proxy_cache uc-cache;
+ # do not consider secure urls as new files
+ proxy_cache_key $scheme$proxy_host$uri;
+ # only one request at a time will be allowed to populate a new cache element
+ proxy_cache_lock on;
+ proxy_cache_min_uses 1;
+ proxy_cache_use_stale updating;
+ # how long should the data be kept in the cache
+ proxy_cache_valid 200 7d;
+ # instruct browser to cache this
+ expires 7d;
+ # headers
+ proxy_ignore_headers Cache-Control;
+ proxy_ignore_headers Set-Cookie;
+ proxy_ignore_headers X-Accel-Expires;
+ proxy_ignore_headers Expires;
+ proxy_hide_header Pragma;
+ add_header X-Cache $upstream_cache_status;
+ }
+ }
+ location / {
+ # only urls to video and audio files are allowed, discard any requested path for other urls
+ rewrite ^/(.*)$ /index.html? break;
+ }
+}
diff --git a/8.Cache/1.Install_cache/0_setup.py b/8.Cache/1.Install_cache/0_setup.py
new file mode 100644
index 00000000..510dd815
--- /dev/null
+++ b/8.Cache/1.Install_cache/0_setup.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import utils
+
+
+def setup(interactive=True):
+ dir_path = utils.get_dir(__file__)
+ rc, hostname = utils.exec_cmd('hostname')
+ if rc != 0 or not hostname:
+ hostname = 'cache'
+ cmds = [
+ 'mkdir -p /var/www/cache',
+ dict(
+ line='write',
+ template='%s/index.html' % dir_path,
+ target='/var/www/cache/index.html',
+ params=dict(hostname=hostname),
+ ),
+ ]
+ utils.run_commands(cmds)
diff --git a/8.Cache/1.Install_cache/0_setup.sh b/8.Cache/1.Install_cache/0_setup.sh
deleted file mode 100755
index a723e053..00000000
--- a/8.Cache/1.Install_cache/0_setup.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-source /root/envsetup/global-conf.sh
-
-sed -i "s@\[CLIENT\]@$(hostname)@" default
-sed -i "s@\[CACHE_DIR\]@${CACHE_DIR}@" default
-sed -i "s@\[CACHE_NAME\]@${CACHE_NAME}@" default
-sed -i "s@\[CACHE_SIZE\]@${CACHE_SIZE}@" default
-sed -i "s@\[CACHE_PROXY_PASS_STREAM\]@${CACHE_PROXY_PASS_STREAM}@" default
-sed -i "s@\[CACHE_PROXY_PASS_VOD\]@${CACHE_PROXY_PASS_VOD}@" default
-sed -i "s@\[CLIENT\]@$(hostname)@" index.html
-
-apt-get purge -y apache2
-apt-get install nginx -y
-mv default /etc/nginx/sites-enabled/default
-mv index.html /usr/share/nginx/html/index.html
-# create crossdomain
-cat << EOF >> /var/www/html/crossdomain.xml
-<cross-domain-policy>
-<allow-access-from domain="*" secure="false"/>
-<site-control permitted-cross-domain-policies="all"/>
-</cross-domain-policy>
-EOF
-chmod 777 /var/www/html/crossdomain.xml
-update-rc.d nginx defaults
-service nginx restart
diff --git a/8.Cache/1.Install_cache/default b/8.Cache/1.Install_cache/default
deleted file mode 100644
index a2594e23..00000000
--- a/8.Cache/1.Install_cache/default
+++ /dev/null
@@ -1,175 +0,0 @@
-proxy_cache_path [CACHE_DIR] levels=1:2 keys_zone=[CACHE_NAME]:10m max_size=[CACHE_SIZE]g inactive=5m;
-
-server {
- listen 80 default_server;
- server_name [CLIENT];
-
- location /nginx_status {
- stub_status on;
- allow 127.0.0.1;
- deny all;
- }
-
- location ~* /crossdomain.xml {
- alias /var/www/html/crossdomain.xml;
- }
-
-# cache configuration for live
- location ~* /live/.*\.(m3u8).*$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_STREAM]; # http only
- # only one request at a time will be allowed to populate a new cache element
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- # instruct browser never to cache this
- expires -1;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- # how long should the data be kept in the cache
- proxy_cache_valid 200 1s;
- # do not consider secure urls as new files
- proxy_cache_key $scheme$proxy_host$uri;
- }
-
- location ~* /live/.*\.(ts).*$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_STREAM]; # http only
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 2s;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- proxy_cache_valid 200 10s;
- # do not consider secure urls as new files
- proxy_cache_key $scheme$proxy_host$uri;
- }
-
-# cache configuration for vod (old easycast stations)
- location ~* /vod/.*\.(m3u8)$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_STREAM]; # http only
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 7d;
- proxy_cache_valid 200 7d;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- # do not consider secure urls as new files
- proxy_cache_key $scheme$proxy_host$uri;
- }
-
- location ~* /vod/.*\.(ts)$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_STREAM]; # http only
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 7d;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- proxy_cache_valid 200 7d;
- # do not consider secure urls as new files
- proxy_cache_key $scheme$proxy_host$uri;
- }
-
-# cache configuration for vod (new easycast stations 06/04/2016)
- location ~* \.(m3u8)$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_VOD]; # http only
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 7d;
- proxy_cache_valid 200 7d;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- # do not consider secure urls as new files
- proxy_cache_key $scheme$proxy_host$uri;
- }
-
- location ~* \.(ts)$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_VOD]; # http only
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 7d;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- proxy_cache_valid 200 7d;
- # do not consider secure urls as new files
- proxy_cache_key $scheme$proxy_host$uri;
- }
-}
-
-server {
- listen 443 ssl;
- server_name [CLIENT];
-
- ssl_certificate /etc/ssl/ubicast_tv/ubicast_tv_bundle.pem;
- ssl_certificate_key /etc/ssl/ubicast_tv/ubicast_tv.key;
-
- location ~* /crossdomain.xml {
- alias /var/www/html/crossdomain.xml;
- }
-
-# cache configuration for vod (25/07/2016)
- location ~* \.(m3u8)$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_VOD];
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 7d;
- proxy_cache_valid 200 7d;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- }
-
- location ~* \.(ts)$ {
- proxy_cache [CACHE_NAME];
- proxy_pass [CACHE_PROXY_PASS_VOD];
- proxy_cache_lock on;
- proxy_ignore_headers Cache-Control;
- proxy_ignore_headers Set-Cookie;
- proxy_ignore_headers X-Accel-Expires;
- proxy_ignore_headers Expires;
- proxy_hide_header Pragma;
- expires 7d;
- add_header X-Cache $upstream_cache_status;
- proxy_cache_use_stale updating;
- proxy_cache_min_uses 1;
- proxy_cache_valid 200 7d;
- }
-}
diff --git a/8.Cache/1.Install_cache/index.html b/8.Cache/1.Install_cache/index.html
index 533c624c..3558358f 100644
--- a/8.Cache/1.Install_cache/index.html
+++ b/8.Cache/1.Install_cache/index.html
@@ -1,16 +1,20 @@
<!DOCTYPE html>
-<html>
-<head>
-<title>Welcome to nginx!</title>
-<style>
- body {
- width: 35em;
- margin: 0 auto;
- font-family: Tahoma, Verdana, Arial, sans-serif;
- }
-</style>
-</head>
-<body>
-<h1>[CLIENT] nginx cache server</h1>
-</body>
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <title>UbiCast cache server</title>
+ <style>
+ html { background: #222; color: #ddd; }
+ body { margin: 0 auto; max-width: 500px; }
+ a { color: #5cf; text-decoration: none; }
+ a:hover { text-decoration: underline; }
+ </style>
+ </head>
+ <body>
+ <h1>UbiCast cache server</h1>
+ <hr/>
+ <p>Hosted on server {{ hostname }}.</p>
+ <br/>
+ <hr/>
+ <p>Powered by UbiCast -- <a href="https://www.ubicast.eu">https://www.ubicast.eu</a></p>
+ </body>
</html>
diff --git a/8.Cache/2.Deploy_munin_configuration/0_setup.sh b/8.Cache/2.Deploy_munin_configuration/0_setup.sh
deleted file mode 100755
index 462bf12f..00000000
--- a/8.Cache/2.Deploy_munin_configuration/0_setup.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-source /root/envsetup/global-conf.sh
-
-# for munin
-DEBIAN_FRONTEND=noninteractive apt-get install -y ubicast-config
diff --git a/8.Cache/3.Install_ferm/0_setup.sh b/8.Cache/2.Install_ferm/0_setup.sh
similarity index 100%
rename from 8.Cache/3.Install_ferm/0_setup.sh
rename to 8.Cache/2.Install_ferm/0_setup.sh
diff --git a/8.Cache/3.Install_ferm/ferm.conf b/8.Cache/2.Install_ferm/ferm.conf
similarity index 100%
rename from 8.Cache/3.Install_ferm/ferm.conf
rename to 8.Cache/2.Install_ferm/ferm.conf
diff --git a/launcher.sh b/launcher.sh
index 46e372f2..8e82689c 100755
--- a/launcher.sh
+++ b/launcher.sh
@@ -61,7 +61,8 @@ campusmanager() {
cache() {
python3 /root/envsetup/envsetup.py 81
python3 /root/envsetup/envsetup.py 82
- python3 /root/envsetup/envsetup.py 83
+ python3 /root/envsetup/envsetup.py 25
+ python3 /root/envsetup/envsetup.py 26
}
tests() {
--
GitLab