Merge branch 't32502-monitoring-via-elastic' into 'master'

Add elastic monitoring tools for benchmark, refs #32502

See merge request mediaserver/envsetup!15

.flake8

@@ -7,6 +7,7 @@ ignore =
     W505
 
 per-file-ignores =
+    roles/elastic.elasticsearch/*:E713
     roles/manager/files/set_site_url.py:E402
     library/*:E402
     library/nmcli.py:E402,F401

.gitignore

@@ -11,6 +11,7 @@ inventories/local*/host_vars/localhost.yml
 inventories/offline*/host_vars/localhost.yml
 playbooks/_*
 roles/_*
+roles/elastic.elasticsearch
 ./logs/
 log/
 

.yamllint

@@ -4,6 +4,7 @@ extends: default
 
 ignore: |
   .venv/
+  roles/elastic.elasticsearch/
 
 rules:
   braces:

Makefile

@@ -5,6 +5,7 @@ PIP_BIN = $(shell command -v $(VENV)/bin/pip3 || command -v pip3 || echo pip3)
 PIP_COMPILE_BIN = $(shell command -v $(VENV)/bin/pip-compile || command -v pip-compile)
 ANSIBLE_BIN = $(shell command -v ansible || command -v $(VENV)/bin/ansible)
 ANSIBLE_PLAYBOOK_BIN = $(shell command -v ansible-playbook || command -v $(VENV)/bin/ansible-playbook)
+ANSIBLE_GALAXY_BIN = $(shell command -v ansible-galaxy || command -v $(VENV)/bin/ansible-galaxy)
 ANSIBLE_LINT_BIN = $(shell command -v ansible-lint || command -v $(VENV)/bin/ansible-lint)
 YAMLLINT_BIN = $(shell command -v yamllint || command -v $(VENV)/bin/yamllint)
 FLAKE8_BIN = $(shell command -v flake8 || command -v $(VENV)/bin/flake8)
@@ -38,6 +39,8 @@ venv:
 install: venv
 	$(PIP_BIN) install -U pip wheel
 	$(PIP_BIN) install -r requirements.txt
+	$(ANSIBLE_GALAXY_BIN) install -r requirements.yml
+
 
 .PHONY: install-dev
 ## install-dev: Install development requirements

playbooks/bench.yml

@@ -11,11 +11,49 @@
   tags: bench_server
   roles: 
     - bench-server
+  tasks:
+    - name: restart bench-server
+      service:
+        name: bench-server
+        state: restarted
+      tags: [ 'never', 'prepare-bench' ]
 
 - name: DEPLOY BENCHMARK WORKERS
   hosts: bench_worker
   tags: bench_worker
   roles: 
     - bench-worker
+  tasks:
+    - name: restart bench-worker
+      service:
+        name: bench-worker
+        state: restarted
+      tags: [ 'never', 'prepare-bench' ]
+
+- name: DEPLOY ELASTIC KIBANA SERVER
+  hosts: elastic
+  vars:
+    - es_heap_size: 2g
+    - es_config:
+        network.host: '{{ hostvars[groups['elastic'][0]]['ansible_default_ipv4']['address'] }}'
+        node.data: true
+        node.master: true
+        cluster.initial_master_nodes: '{{ hostvars[groups["elastic"][0]].ansible_hostname }}'
+    - kibana_server_host: "{{ hostvars[groups['elastic'][0]]['ansible_default_ipv4']['address'] }}"
+    - elastic_host: "{{ es_config['network.host'] }}"
+    - es_api_host: "{{ es_config['network.host'] }}"
+  tags: [ 'never', 'monbench' ]
+  roles:
+    - elastic
+
+- name: DEPLOY METRICBEAT WORKERS
+  hosts: mediaserver,postgres
+  tags: [ 'never', 'monbench' ]
+  vars:
+    - kibana_server_host: "{{ hostvars[groups['elastic'][0]]['ansible_default_ipv4']['address'] }}"
+    - elastic_host: "{{ es_config['network.host'] }}"
+    - es_api_host: "{{ es_config['network.host'] }}"
+  roles:
+    - metricbeat
 
 ...

requirements.yml

+---
+- src: elastic.elasticsearch
+  version: 7.8.1
+
+...

roles/elastic/defaults/main.yml

+---
+kibana_default_port: 5601
+kibana_server_host: localhost
+
+...

roles/elastic/handlers/main.yml

+---
+- name: restart kibana
+  service:
+    name: kibana
+    state: restarted
+
+- name: restart apm-server
+  service:
+    name: apm-server
+    state: restarted
+
+...

roles/elastic/meta/main.yml

+---
+dependencies:
+  - role: elastic.elasticsearch
+
+...

roles/elastic/tasks/main.yml

+---
+- name: install kibana package
+  apt:
+    pkg:
+      - kibana
+    state: latest
+
+- name: deploy kibana configuration
+  template:
+    src: kibana.yml.j2
+    dest: /etc/kibana/kibana.yml
+  notify: restart kibana
+
+- name: install apm-server package
+  apt:
+    pkg:
+      - apm-server
+    state: latest
+
+- name: deploy apm-server configuration
+  template:
+    src: apm-server.yml.j2
+    dest: /etc/apm-server/apm-server.yml
+  notify: restart apm-server
+
+...

roles/elastic/templates/kibana.yml.j2

+# Kibana is served by a back end server. This setting specifies the port to use.
+server.port: {{ kibana_default_port }}
+
+# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
+# The default is 'localhost', which usually means remote machines will not be able to connect.
+# To allow connections from remote users, set this parameter to a non-loopback address.
+#server.host: "localhost"
+server.host: {{ kibana_server_host }}
+
+# Enables you to specify a path to mount Kibana at if you are running behind a proxy.
+# Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath
+# from requests it receives, and to prevent a deprecation warning at startup.
+# This setting cannot end in a slash.
+# server.basePath: ""
+
+# Specifies whether Kibana should rewrite requests that are prefixed with
+# `server.basePath` or require that they are rewritten by your reverse proxy.
+# This setting was effectively always `false` before Kibana 6.3 and will
+# default to `true` starting in Kibana 7.0.
+#server.rewriteBasePath: false
+
+# The maximum payload size in bytes for incoming server requests.
+#server.maxPayloadBytes: 1048576
+
+# The Kibana server's name.  This is used for display purposes.
+#server.name: "your-hostname"
+
+# The URLs of the Elasticsearch instances to use for all your queries.
+elasticsearch.hosts: ["http://{{ elastic_host }}:9200"]
+
+# When this setting's value is true Kibana uses the hostname specified in the server.host
+# setting. When the value of this setting is false, Kibana uses the hostname of the host
+# that connects to this Kibana instance.
+#elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
+# dashboards. Kibana creates a new index if the index doesn't already exist.
+#kibana.index: ".kibana"
+
+# The default application to load.
+#kibana.defaultAppId: "home"
+
+# If your Elasticsearch is protected with basic authentication, these settings provide
+# the username and password that the Kibana server uses to perform maintenance on the Kibana
+# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
+# is proxied through the Kibana server.
+#elasticsearch.username: "kibana_system"
+#elasticsearch.password: "pass"
+
+# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
+# These settings enable SSL for outgoing requests from the Kibana server to the browser.
+#server.ssl.enabled: false
+#server.ssl.certificate: /path/to/your/server.crt
+#server.ssl.key: /path/to/your/server.key
+
+# Optional settings that provide the paths to the PEM-format SSL certificate and key files.
+# These files are used to verify the identity of Kibana to Elasticsearch and are required when
+# xpack.security.http.ssl.client_authentication in Elasticsearch is set to required.
+#elasticsearch.ssl.certificate: /path/to/your/client.crt
+#elasticsearch.ssl.key: /path/to/your/client.key
+
+# Optional setting that enables you to specify a path to the PEM file for the certificate
+# authority for your Elasticsearch instance.
+#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
+
+# To disregard the validity of SSL certificates, change this setting's value to 'none'.
+#elasticsearch.ssl.verificationMode: full
+
+# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
+# the elasticsearch.requestTimeout setting.
+#elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
+# must be a positive integer.
+#elasticsearch.requestTimeout: 30000
+
+# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
+# headers, set this value to [] (an empty list).
+#elasticsearch.requestHeadersWhitelist: [ authorization ]
+
+# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
+# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
+#elasticsearch.customHeaders: {}
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
+#elasticsearch.shardTimeout: 30000
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
+#elasticsearch.startupTimeout: 5000
+
+# Logs queries sent to Elasticsearch. Requires logging.verbose set to true.
+#elasticsearch.logQueries: false
+
+# Specifies the path where Kibana creates the process ID file.
+#pid.file: /var/run/kibana.pid
+
+# Enables you specify a file where Kibana stores log output.
+#logging.dest: stdout
+
+# Set the value of this setting to true to suppress all logging output.
+#logging.silent: false
+
+# Set the value of this setting to true to suppress all logging output other than error messages.
+#logging.quiet: false
+
+# Set the value of this setting to true to log all events, including system usage information
+# and all requests.
+#logging.verbose: false
+
+# Set the interval in milliseconds to sample system and process performance
+# metrics. Minimum is 100ms. Defaults to 5000.
+#ops.interval: 5000
+
+# Specifies locale to be used for all localizable strings, dates and number formats.
+# Supported languages are the following: English - en , by default , Chinese - zh-CN .
+#i18n.locale: "en"

roles/metricbeat/defaults/main.yml

+---
+elastic_host: localhost
+elastic_port: 9200
+kibana_server_host: localhost
+
+...

roles/metricbeat/handlers/main.yml

+---
+- name: restart metricbeat
+  service:
+    name: metricbeat
+    state: restarted
+
+...

roles/metricbeat/tasks/main.yml

+---
+- name: install apt-transport-https
+  apt:
+    name: apt-transport-https
+    state: latest
+
+- name: install elastic GPG key
+  apt_key:
+    url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+    state: present
+
+- name: install elastic repository
+  apt_repository:
+    repo: deb https://artifacts.elastic.co/packages/7.x/apt stable main
+
+- name: install metricbeat
+  apt:
+    name: metricbeat
+    state: latest
+
+- name: install metricbeat configuration
+  template:
+    src: metricbeat.yml.j2
+    dest: /etc/metricbeat/metricbeat.yml
+  notify: restart metricbeat
+
+- name: enable metricbeat dashboard
+  command: metricbeat setup
+  when: inventory_hostname == groups['mediaserver'][0]
+
+- name: enable sql metricbeat configuration
+  template:
+    src: postgresql.yml.j2
+    dest: /etc/metricbeat/modules.d/postgresql.yml
+  when: "'postgres' in group_names"
+  notify: restart metricbeat
+
+...

roles/metricbeat/templates/metricbeat.yml.j2

+###################### Metricbeat Configuration Example #######################
+
+# This file is an example configuration file highlighting only the most common
+# options. The metricbeat.reference.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/metricbeat/index.html
+
+# =========================== Modules configuration ============================
+
+metricbeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
+
+
+# ================================== General ===================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+# ================================= Dashboards =================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here or by using the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+# =================================== Kibana ===================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  host: "{{ kibana_server_host }}:5601"
+
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
+# =============================== Elastic Cloud ================================
+
+# These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+# ================================== Outputs ===================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["{{ elastic_host }}:{{ elastic_port }}"]
+
+  # Protocol - either `http` (default) or `https`.
+  #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
+  #username: "elastic"
+  #password: "changeme"
+
+# ------------------------------ Logstash Output -------------------------------
+#output.logstash:
+  # The Logstash hosts
+  #hosts: ["localhost:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+# ================================= Processors =================================
+
+# Configure processors to enhance or manipulate events generated by the beat.
+
+processors:
+  - add_host_metadata: ~
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+
+
+# ================================== Logging ===================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+#logging.selectors: ["*"]
+
+# ============================= X-Pack Monitoring ==============================
+# Metricbeat can export internal metrics to a central Elasticsearch monitoring
+# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
+# reporting is disabled by default.
+
+# Set to true to enable the monitoring reporter.
+#monitoring.enabled: false
+
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
+# Uncomment to send the metrics to Elasticsearch. Most settings from the
+# Elasticsearch output are accepted here as well.
+# Note that the settings should point to your Elasticsearch *monitoring* cluster.
+# Any setting that is not set is automatically inherited from the Elasticsearch
+# output configuration, so if you have the Elasticsearch output configured such
+# that it is pointing to your Elasticsearch monitoring cluster, you can simply
+# uncomment the following line.
+#monitoring.elasticsearch:
+
+# ================================= Migration ==================================
+
+# This allows to enable 6.7 migration aliases
+#migration.6_to_7.enabled: true
\ No newline at end of file

roles/metricbeat/templates/postgresql.yml.j2

+# Module: postgresql
+# Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.8/metricbeat-module-postgresql.html
+
+- module: postgresql
+  #metricsets:
+  #  - database
+  #  - bgwriter
+  #  - activity
+  period: 10s
+  hosts: ["postgres://localhost:5432"]
+  username: {{ metricbeat_pgsql_username }}
+  password: {{ metricbeat_pgsql_pass }}