Commit 104b4ad1 authored by Stéphane Diemer's avatar Stéphane Diemer
Browse files

Use new client | refs #29074

parent c9368bb0
Pipeline #4231 passed with stage
in 19 seconds
flake8:
image: python:3-alpine
tags:
- docker
before_script:
- python -m pip install --upgrade pip
- pip3 install flake8
script:
# Run flake8 (pycodestyle + pyflakes) check.
# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
# Ignored errors:
# - E128: continuation line under-indented for visual indent
# - E501: line too long
# - E731: do not assign a lambda expression, use a def
# - W504: line break after binary operator (conflicts with old pep8 package)
# - W505: doc line too long
- flake8 --ignore=E128,E501,E731,W504,W505 .
......@@ -14,11 +14,6 @@ To start a live stream on Ubuntu 16.04:
apt-get install gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-plugins-bad gstreamer1.0-tools gstreamer1.0-x
```
To load the settings from an instance:
```bash
apt-get install python3-mysqldb
```
## Available scripts
......@@ -37,8 +32,3 @@ apt-get install python3-mysqldb
* ms_ip_white_list_tester
Get all routing rules, extract IP addresses from them and test that each IP is in Nginx secure link white list.
## Notes
The MediaServer API client is downloaded from GitHub at first run.
{
"SERVER_URL": "https://my.mediaserver.net",
"API_KEY": "my-api-key",
"CLIENT_ID": "python-api-client",
"PROXIES": {
"http": "",
"https": ""
},
"UPLOAD_CHUNK_SIZE": 5242880,
"VERIFY_SSL": false,
"OID": "l125620f00eact2dmj2r"
}
Subproject commit 9adfa4e7eb7516a0f2645c958f5877226eedb61b
Subproject commit 0ec5811f6b314cad266c77cc295c2c0ee34c9b21
......@@ -27,7 +27,7 @@ if __name__ == '__main__':
msc = init_client(logger.name, sys.argv[1] if len(sys.argv) > 1 else None, CONFIG_DEFAULT)
# test local MS IP
ms_host = msc.config['SERVER_URL'].split('://')[-1].split('/')[0].split(':')[0]
ms_host = msc.conf['SERVER_URL'].split('://')[-1].split('/')[0].split(':')[0]
ms_ip = socket.gethostbyname(ms_host)
if ms_ip != '127.0.0.1':
logger.error('The MediaServer IP must be 127.0.0.1 (currently %s).\nPlease check "/etc/hosts" content.', ms_ip)
......
......@@ -27,18 +27,18 @@ def get_aac_encoder():
def start_live(msc, run_pipeline=True):
# start live stream
if not msc.config.get('OID'):
if not msc.conf.get('OID'):
raise ValueError('No value defined for OID in config.')
logger.info('Starting gst-launch-1.0')
caps_template = '"video/x-raw, format=(string)I420, width=(int){width}, height=(int){height}, framerate=(fraction){framerate}/1"'
src_config = {'width': 1920, 'height': 1080, 'framerate': 30}
src_caps = caps_template.format(**src_config)
pipeline = 'gst-launch-1.0 videotestsrc is-live=true pattern=%s ! %s ! tee name=tee' % (msc.config.get('PATTERN', 0), src_caps)
pipeline = 'gst-launch-1.0 videotestsrc is-live=true pattern=%s ! %s ! tee name=tee' % (msc.conf.get('PATTERN', 0), src_caps)
encoder_branch_template = 'tee. ! queue ! videorate ! videoscale ! {caps} ! queue ! timeoverlay text="{height}@{video_bitrate_kbps}k" ! queue ! x264enc tune=zerolatency speed-preset=ultrafast bitrate={video_bitrate_kbps} key-int-max={framerate} ! flvmux streamable=true name=mux{height} ! rtmpsink location={publish_uri} audiotestsrc is-live=true wave=ticks ! queue ! {aac_encoder} bitrate={audio_bitrate} ! mux{height}.'
randomize = 'yes' if msc.config.get('RANDOMIZE') else 'no'
logger.info('Preparing live on MS "%s/permalink/%s"', msc.config['SERVER_URL'], msc.config['OID'])
resp = msc.api('lives/prepare/', method='post', data={'oid': msc.config['OID'], 'multi_streams': 'yes', 'randomize': randomize})
randomize = 'yes' if msc.conf.get('RANDOMIZE') else 'no'
logger.info('Preparing live on MS "%s/permalink/%s"', msc.conf['SERVER_URL'], msc.conf['OID'])
resp = msc.api('lives/prepare/', method='post', data={'oid': msc.conf['OID'], 'multi_streams': 'yes', 'randomize': randomize})
logger.info(resp)
for stream in resp['streams']:
caps = caps_template.format(**stream)
......@@ -47,9 +47,9 @@ def start_live(msc, run_pipeline=True):
stream['aac_encoder'] = get_aac_encoder()
encoder_branch = encoder_branch_template.format(**stream)
pipeline += ' %s' % encoder_branch
logger.info('Starting live on MS "%s/permalink/%s"', msc.config['SERVER_URL'], msc.config['OID'])
logger.info('Starting live on MS "%s/permalink/%s"', msc.conf['SERVER_URL'], msc.conf['OID'])
logger.info('Pipeline: %s' % pipeline)
resp = msc.api('lives/start/', method='post', data={'oid': msc.config['OID']})
resp = msc.api('lives/start/', method='post', data={'oid': msc.conf['OID']})
logger.info(resp)
logger.debug(pipeline)
if run_pipeline:
......@@ -59,17 +59,17 @@ def start_live(msc, run_pipeline=True):
def stop_live(msc):
need_stop = False
if msc.config.get('DELETE_LIVE_AT_END', True):
logger.info('Deleting live "%s"' % msc.config['OID'])
response = msc.api('medias/delete/', method='post', data={'oid': msc.config['OID'], 'delete_metadata': 'yes', 'delete_resources': 'yes'})
if msc.conf.get('DELETE_LIVE_AT_END', True):
logger.info('Deleting live "%s"' % msc.conf['OID'])
response = msc.api('medias/delete/', method='post', data={'oid': msc.conf['OID'], 'delete_metadata': 'yes', 'delete_resources': 'yes'})
if not response.get('success'):
logger.error('Failed to delete media in MS:\n%s' % response)
need_stop = True
else:
need_stop = True
if need_stop:
logger.info('Stopping live "%s"' % msc.config['OID'])
resp = msc.api('lives/stop/', method='post', data={'oid': msc.config['OID']})
logger.info('Stopping live "%s"' % msc.conf['OID'])
resp = msc.api('lives/stop/', method='post', data={'oid': msc.conf['OID']})
logger.info(resp)
......@@ -81,12 +81,12 @@ if __name__ == '__main__':
# get ms client
msc = init_client(logger.name, sys.argv[1] if len(sys.argv) > 1 else None, CONFIG_DEFAULT)
if not msc.config.get('OID'):
if not msc.conf.get('OID'):
# get or create live page
live = get_or_create_live(msc)
logger.info('No OID defined in config, determining one automatically that will be automatically removed after stopping')
msc.config['OID'] = live['oid']
msc.config['DELETE_LIVE_AT_END'] = True
msc.conf['OID'] = live['oid']
msc.conf['DELETE_LIVE_AT_END'] = True
start_live(msc)
stop_live(msc)
......@@ -39,10 +39,10 @@ if __name__ == '__main__':
# get or create live page
live = get_or_create_live(msc)
msc.config['OID'] = live['oid']
msc.conf['OID'] = live['oid']
# start live stream in another process
if msc.config['MANAGE_LIVE']:
if msc.conf['MANAGE_LIVE']:
pipeline = ms_live_streamer.start_live(msc, run_pipeline=False)
pipeline_process = subprocess.Popen(pipeline, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, shell=True, preexec_fn=os.setsid)
time.sleep(15) # let some time for stream init
......@@ -57,7 +57,7 @@ if __name__ == '__main__':
checked = 0
failures = 0
try:
checked, failures = test_media_resources(msc, msc.config['OID'])
checked, failures = test_media_resources(msc, msc.conf['OID'])
except Exception:
logger.info('Failed to check resources.')
exit_code = 1
......@@ -66,7 +66,7 @@ if __name__ == '__main__':
exit_code = 1
logger.info('')
finally:
if msc.config['MANAGE_LIVE']:
if msc.conf['MANAGE_LIVE']:
time.sleep(1)
ms_live_streamer.stop_live(msc)
try:
......
......@@ -2,96 +2,46 @@
# -*- coding: utf-8 -*-
# Copyright 2017, Stéphane Diemer
import imp
import logging
import os
import re
import subprocess
import sys
logger = logging.getLogger('ms_testing_lib.client_functions')
MS_CLIENT_PATH = '../ms_client/mediaserver_api_client.py'
MS_CLIENT_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'ms_client')
def init_client(client_id='tester', target=None, extra_conf=None):
# target can be either a system user with a MediaServer instance or a path to a configuration file.
ms_user = None
conf_path = None
cleaned_target = None
if target:
if os.path.isfile(target):
conf_path = target
logger.info('Using configuration file "%s".', conf_path)
cleaned_target = target
logger.info('Using configuration file "%s".', target)
else:
ms_user = target
if target == 'self':
ms_user = subprocess.getoutput('whoami').strip()
else:
ms_user = target
cleaned_target = 'unix:%s' % ms_user
logger.info('Will build configuration for local user %s' % ms_user)
elif os.path.isfile('config.json'):
logger.info('No target configuration or user specified, will use config.json')
else:
logger.info('No target configuration or user specified, will use default configuration')
# load open source client lib
client_path = os.path.join(os.path.dirname(__file__), MS_CLIENT_PATH)
if not os.path.exists(client_path):
raise Exception('The open source client file does not exists. Please initialize and update the testing suite sub modules.')
ms_client = imp.load_source('ms_client', client_path)
if MS_CLIENT_DIR not in sys.path:
sys.path.append(MS_CLIENT_DIR)
from ms_client.client import MediaServerClient
if extra_conf:
ms_client.CONFIG_DEFAULT.update(extra_conf)
MediaServerClient.DEFAULT_CONF = extra_conf
# init client
msc = ms_client.MediaServerClient(conf_path)
msc.config['CLIENT_ID'] = client_id
level = getattr(logging, msc.config.get('LOG_LEVEL') or 'INFO')
msc = MediaServerClient(cleaned_target)
msc.conf['CLIENT_ID'] = client_id
level = getattr(logging, msc.conf.get('LOG_LEVEL') or 'INFO')
logging.root.setLevel(level)
urllib3_logger = logging.getLogger('requests.packages.urllib3')
urllib3_logger.setLevel(logging.WARNING)
# get MS user
if ms_user == 'self':
ms_user = subprocess.getoutput('whoami').strip()
if ms_user:
# override config with data from user db
instance_dir = '/home/%s/msinstance' % ms_user
if not os.path.exists(instance_dir):
logger.error('Instance dir "%s" does not exists.', instance_dir)
sys.exit(1)
logger.info('Retrieving configuration from MS user "%s".', ms_user)
# get MS db settings
# do not load entire module to avoid errors
mssettings_path = os.path.join(instance_dir, 'conf', 'mssettings.py')
with open(mssettings_path, 'r') as fo:
content = fo.read().replace('\r', '')
if 'DATABASES' not in content:
logger.error('Failed to get instance db settings, no configuration found in mssettings.')
sys.exit(1)
content = content[content.index('\nDATABASES'):]
content = content[:content.index('\n}') + 2]
dbs_module = imp.new_module('dbs_module')
exec(content, dbs_module.__dict__)
# get MS site settings
cmd = 'python3 %s shell -i python' % os.path.join(instance_dir, 'manage.py')
if os.environ.get('USER') == ms_user:
cmd = ['bash', '-c', cmd]
else:
cmd = ['su', ms_user, '-c', cmd]
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate(input=('from mediaserver.main.models import SiteSettings; ms_ss=SiteSettings.get_singleton(); print("site_url:%s" % ms_ss.site_url); print("master_api_key:%s" % ms_ss.master_api_key); print("resources_secret:%s" % ms_ss.resources_secret);').encode('utf-8'))
out = out.decode('utf-8') if out else ''
if err:
out += '\n' + err.decode('utf-8')
m = re.match(r'site_url:(.*)\nmaster_api_key:(.*)\nresources_secret:(.*)', out)
if not m:
logger.error('Failed to get instance settings:\n%s', out)
sys.exit(1)
site_url, master_api_key, resources_secret = m.groups()
# change client config
msc.config['SECURE_LINK'] = True if resources_secret else False
msc.config['SERVER_URL'] = site_url
msc.config['API_KEY'] = master_api_key
logger.debug('Config is: %s', msc.config)
logger.debug('Configuration is: %s', msc.conf)
return msc
......@@ -105,5 +55,5 @@ def get_or_create_live(msc, slug='livestreaming-tester', title='Live for testing
if not live:
resp = msc.api('medias/add/', method='post', data={'type': 'live', 'slug': slug, 'title': title})
live = resp
logger.info('Using live %s: "%s/permalink/%s/".', slug, msc.config['SERVER_URL'], live['oid'])
logger.info('Using live %s: "%s/permalink/%s/".', slug, msc.conf['SERVER_URL'], live['oid'])
return live
......@@ -118,8 +118,8 @@ def test_media_resources(msc, media_oid):
logger.info('\033[94m---- Testing rule "%s" from %s redirecting to "%s" (proxy: %s) \033[0m', rule['name'], rule['from_ip'], rule['reroute_to'], rule.get('use_proxy', True))
headers = {'from-ip': rule['from_ip'].split('/')[0]}
# test access from MS and from each host of routing rules
resp1 = msc.api('medias/modes/', method='get', headers=headers, params={'oid': media_oid, 'html5': msc.config['FORMATS']})
resp2 = msc.api('medias/modes/', method='get', headers=headers, params={'oid': media_oid, 'html5': msc.config['FORMATS']})
resp1 = msc.api('medias/modes/', method='get', headers=headers, params={'oid': media_oid, 'html5': msc.conf['FORMATS']})
resp2 = msc.api('medias/modes/', method='get', headers=headers, params={'oid': media_oid, 'html5': msc.conf['FORMATS']})
if not resp1.get('names'):
logger.error('\033[91m No resources available in first modes call. \033[0m')
failures += 1
......@@ -148,7 +148,7 @@ def test_media_resources(msc, media_oid):
logger.info('\033[90m "%s": skipped \033[0m| Caching of adaptative playlist provided by MS is not supported for now.', res_url)
continue
test_config = dict(msc.config)
test_config = dict(msc.conf)
# test_config['PROXIES'] = None means using the system proxy
# do not use proxy for loop without routing (direct)
test_config['PROXIES'] = None if not rule or rule.get('use_proxy', True) else {'http': '', 'https': ''}
......
......@@ -8,16 +8,20 @@ import sys
# MS testing suite
from ms_testing_lib.client_functions import init_client
logger = logging.getLogger('ms_live_viewers_count')
def poll_live_viewers(msc):
d = msc.api('/lives/get-viewers', method='get', data={'oid': msc.config['OID']})
if d.get('success'):
viewers_count = d['viewers']
logger.info("Viewers count: %s" % viewers_count)
logger = logging.getLogger('ms_viewers_count')
def poll_viewers(msc, oid):
data = dict(oid=oid) if oid else None
response = msc.api('stats/viewers/', method='get', data=data)
if response.get('success'):
logger.info('Viewers count: %s, viewers peak: %s (%s)' % (
response.get('current_viewers'),
response.get('viewers_peak'),
response.get('viewers_peak_date')
))
else:
logger.error('Error: %s' % d)
logger.error('Error: %s' % response)
return True
......@@ -29,6 +33,7 @@ if __name__ == '__main__':
# get ms client
msc = init_client(logger.name, sys.argv[1] if len(sys.argv) > 1 else None)
gobject.timeout_add_seconds(1, poll_live_viewers, msc)
oid = sys.argv[2] if len(sys.argv) > 2 else None
gobject.timeout_add_seconds(1, poll_viewers, msc, oid)
ml = gobject.MainLoop()
ml.run()
......@@ -33,18 +33,18 @@ if __name__ == '__main__':
# get ms client
msc = init_client(logger.name, sys.argv[1] if len(sys.argv) > 1 else None, CONFIG_DEFAULT)
msc.config['FORMATS'] = 'mp4_webm_m3u8'
msc.conf['FORMATS'] = 'mp4_webm_m3u8'
# create new media
test_file = (sys.argv[2] if len(sys.argv) > 2 else None) or msc.config['TEST_FILE']
test_file = (sys.argv[2] if len(sys.argv) > 2 else None) or msc.conf['TEST_FILE']
file_path = os.path.join(os.path.dirname(__file__), test_file)
logger.info('Uploading file "%s".' % test_file)
response = msc.add_media('VOD for testing script', file_path=file_path, slug=msc.config['VOD_SLUG'])
response = msc.add_media('VOD for testing script', file_path=file_path, slug=msc.conf['VOD_SLUG'])
if not response.get('success'):
logger.info('Failed to add media in MS:\n%s' % response)
sys.exit(1)
vod_oid = response['oid']
logger.info('Using vod %s: "%s/permalink/%s/".', msc.config['VOD_SLUG'], msc.config['SERVER_URL'], vod_oid)
logger.info('Using vod %s: "%s/permalink/%s/".', msc.conf['VOD_SLUG'], msc.conf['SERVER_URL'], vod_oid)
exit_code = 0
checked = 0
......
......@@ -21,6 +21,7 @@ logger = logging.getLogger('ms_init_line_time')
viewers = 0
def get_timestamp():
return time.strftime("%Y-%m-%d_%H-%M-%S")
......@@ -36,6 +37,7 @@ def write_line(line):
header = '#' + '\t'.join(('time', 'viewers', 'init_time_ms', 'page_ms', 'modes_ms', 'frags_ms'))
write_line(header)
def write_csv_line(begin_ts, viewers_count, init_time_ms, page_init, resources_init, fragments_init):
begin_date = datetime.datetime.fromtimestamp(begin_ts).strftime('%d/%m/%Y %H:%M:%S')
line = '\t'.join((begin_date, str(viewers_count), str(int(init_time_ms)), str(int(page_init)), str(int(resources_init)), str(int(fragments_init))))
......@@ -85,12 +87,12 @@ class MsPing:
def poll_init_time(self):
self.begin_ts = time.time()
pageurl = self.msc.config['SERVER_URL'] + '/permalink/' + self.msc.config['OID']
pageurl = self.msc.conf['SERVER_URL'] + '/permalink/' + self.msc.conf['OID']
logger.info('Starting page opening on %s' % pageurl)
requests.get(pageurl, verify=False)
after_page_ts = time.time()
self.page_took = 1000 * (after_page_ts - self.begin_ts)
resources = self.msc.api('medias/modes', method='get', params={'oid': self.msc.config['OID'], 'html5': 'webm_ogg_ogv_oga_mp4_mp3_m3u8'})
resources = self.msc.api('medias/modes', method='get', params={'oid': self.msc.conf['OID'], 'html5': 'webm_ogg_ogv_oga_mp4_mp3_m3u8'})
after_resources_ts = time.time()
self.resources_took = 1000 * (after_resources_ts - after_page_ts)
# it takes ~700ms for the js player to init
......@@ -132,7 +134,7 @@ class MsPing:
def poll_live_viewers(self):
global viewers
d = self.msc.api('/lives/get-viewers', method='get', params={'oid': self.msc.config['OID']})
d = self.msc.api('/lives/get-viewers', method='get', params={'oid': self.msc.conf['OID']})
if d.get('success'):
viewers = d['viewers']
else:
......@@ -152,7 +154,12 @@ if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format=log_format)
# get ms client
oid = sys.argv[2] if len(sys.argv) > 2 else None
if not oid:
print('No oid given in command line.\nUsage: %s <unix user or conf path> <oid>' % __file__)
sys.exit(1)
msc = init_client(logger.name, sys.argv[1] if len(sys.argv) > 1 else None)
msc.conf['OID'] = oid
p = MsPing(msc)
......
......@@ -31,6 +31,7 @@ NAME_GROUPS = [
def get_timestamp():
return time.strftime("%Y-%m-%d_%H-%M-%S")
CSV_FNAME = 'mediaservertop-' + get_timestamp() + '.csv'
......@@ -299,6 +300,7 @@ class MediaServerTop:
color = RED
return color
if __name__ == '__main__':
# init logging
log_format = '%(asctime)s %(message)s'
......
......@@ -10,6 +10,7 @@ def parse_header(line):
fields = line.split()
return fields
HR_INTERVAL = 20
last_hr_time = 0
last_hr_viewers = None
......@@ -62,6 +63,7 @@ for index, line in enumerate(fileinput.input()):
def get_real_time(time):
return start_date + datetime.timedelta(seconds=time)
print('Live opening time: %s (%ss, already %s viewers)' % (get_real_time(open_live_time), open_live_time, open_live_viewers))
print('Live closing time: %s' % get_real_time(closed_live_time))
print('Max viewers: %s at %s after %i min' % (max_viewers, get_real_time(max_viewers_time), max_viewers_time / 60))
......
......@@ -61,6 +61,7 @@ def test_routing(msc):
tested.append(reroute_to)
return all_ok
if __name__ == '__main__':
# init logging
log_format = '%(asctime)s %(name)s %(levelname)s %(message)s'
......
......@@ -34,7 +34,7 @@ def get_aac_encoder():
def start_live(msc):
# get or create live page
live = get_or_create_live(msc)
msc.config['OID'] = live['oid']
msc.conf['OID'] = live['oid']
logger.info('Starting gst-launch-1.0')
caps_template = '"video/x-raw, format=(string)I420, width=(int){width}, height=(int){height}, framerate=(fraction){framerate}/1"'
......@@ -43,7 +43,7 @@ def start_live(msc):
pipeline = 'gst-launch-1.0 videotestsrc is-live=true num-buffers=30 ! %s ! tee name=tee' % src_caps
encoder_branch_template = 'tee. ! queue ! videorate ! videoscale ! {caps} ! queue ! timeoverlay text="{height}@{video_bitrate_kbps}k" ! queue ! x264enc tune=zerolatency speed-preset=ultrafast bitrate={video_bitrate_kbps} key-int-max={framerate} ! flvmux streamable=true name=mux{height} ! rtmpsink location={publish_uri}'
resp = msc.api('lives/prepare/', method='post', data={'oid': msc.config['OID'], 'multi_streams': 'yes'})
resp = msc.api('lives/prepare/', method='post', data={'oid': msc.conf['OID'], 'multi_streams': 'yes'})
for stream in resp['streams']:
server_uri = stream['server_uri']
# server_uri = rtmp://delltest.ubicast.net/live/_definst_?doPublish=gOh9n9LX3EybC9/
......@@ -61,8 +61,8 @@ def start_live(msc):
stream['aac_encoder'] = get_aac_encoder()
encoder_branch = encoder_branch_template.format(**stream)
pipeline += ' %s' % encoder_branch
logger.info('Starting live on MS "%s"', msc.config['SERVER_URL'])
resp = msc.api('lives/start/', method='post', data={'oid': msc.config['OID']})
logger.info('Starting live on MS "%s"', msc.conf['SERVER_URL'])
resp = msc.api('lives/start/', method='post', data={'oid': msc.conf['OID']})
logger.info(resp)
logger.debug(pipeline)
status = os.system(pipeline)
......@@ -75,8 +75,8 @@ def start_live(msc):
def stop_live(msc):
logger.info('Stopping live on MS "%s"', msc.config['SERVER_URL'])
resp = msc.api('lives/stop/', method='post', data={'oid': msc.config['OID']})
logger.info('Stopping live on MS "%s"', msc.conf['SERVER_URL'])
resp = msc.api('lives/stop/', method='post', data={'oid': msc.conf['OID']})
logger.info(resp)
......
......@@ -25,7 +25,7 @@ def get_video_track(i):
for i in d:
height = str(get_video_track(i)['height']) + 'p'
#Fichier Bitrate (mbits/s) Framerate réel Synchro (ms) Durée (s)
# File Bitrate (mbits/s) Real framerate Synchro (ms) Duration (s)
fields = [
i["format"]["filename"],
height,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment