From 8fda1fb601bf961c88ef0138054bdd565e07d8e4 Mon Sep 17 00:00:00 2001 From: Roman Vynar Date: Fri, 23 Sep 2016 15:10:24 +0300 Subject: [PATCH] 1.0.5 release. * New tool: Orchestrator - a MySQL replication topology management and visualization tool. Available by /orchestrator URL. * Added ProxySQL metrics and dashboard. * Prometheus 1.1.3. * Consul 0.7.0. * Grafana data can be stored on data container to preserve your custom dashboards and settings. The predefined dashboards are still the subject of upgrade on new versions. * Changed metric storage encoding to achieve less disk space usage by 50-70%. * Re-use MySQL instance with saved data on Query Analytics when re-added through pmm-admin with the same name. * Various dashboard improvements. * Fixed rare issue when nginx tries to use IPv6 for localhost connections. * Improvements and fixes to Query Analytics. --- CHANGELOG.md | 17 ++++- Dockerfile | 82 +++++++++++------------ VERSION | 2 +- docker-compose.yml | 5 +- docker-push.sh | 2 +- entrypoint.sh | 21 +++++- grafana-postinstall.sh | 2 + import-dashboards.py | 87 ++++++++++++++++++------ landing-page/index.html | 45 +++++++------ nginx.conf | 15 +++-- orchestrator.conf.json | 145 ++++++++++++++++++++++++++++++++++++++++ prometheus.yml | 18 +++++ qan-install.sh | 6 +- supervisord.conf | 40 +++++++++-- 14 files changed, 387 insertions(+), 100 deletions(-) create mode 100644 orchestrator.conf.json diff --git a/CHANGELOG.md b/CHANGELOG.md index a5bdac193..961c70db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ Percona Monitoring and Management (PMM) Server +v1.0.5 released 2016-10-14 + +* New tool: Orchestrator - a MySQL replication topology management and visualization tool. Available by /orchestrator URL. +* Added ProxySQL metrics and dashboard. +* Prometheus 1.1.3. +* Consul 0.7.0. +* Grafana data can be stored on data container to preserve your custom dashboards and settings. The predefined dashboards are still the subject of upgrade on new versions. +* Changed metric storage encoding to achieve less disk space usage by 50-70%. +* Re-use MySQL instance with saved data on Query Analytics when re-added through pmm-admin with the same name. +* Various dashboard improvements. +* Fixed rare issue when nginx tries to use IPv6 for localhost connections. +* Improvements and fixes to Query Analytics. + v1.0.4 released 2016-09-13 * Grafana 3.1.1. @@ -17,7 +30,7 @@ v1.0.4 released 2016-09-13 v1.0.3 released 2016-08-05 -* Fixed math for query metrics on QAN App. +* Fixed math for query metrics on Query Analytics. v1.0.2 released 2016-07-28 @@ -42,7 +55,7 @@ v1.0.1 released 2016-06-09 * Added MongoDB dashboards. * Replaced prom-config-api with Consul 0.6.4. * Eliminated most of the ports for PMM server container (now only two: 9001 and configurable 80). -* Added "Server Summary" with aggregated query metrics to QAN app. +* Added "Server Summary" with aggregated query metrics to Query Analytics. * MySQL dashboard updates, added "MySQL InnoDB Metrics Advanced" dashboard. * Added options to configure Prometheus memory and retention. diff --git a/Dockerfile b/Dockerfile index 56cf48fff..0909a3791 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,88 +1,86 @@ -FROM ubuntu:latest +FROM ubuntu:14.04 EXPOSE 80 443 WORKDIR /opt -# ########################### # -# MySQL and other system pkgs # -# ########################### # - -RUN apt-get -y update && apt-get install -y \ - apt-transport-https \ - curl \ - git \ - unzip \ - nginx \ - mysql-server \ - python \ - python-requests \ - supervisor && \ +# ############### # +# System packages # +# ############### # +RUN apt-get -y update && \ + apt-get install -y apt-transport-https curl git unzip nginx mysql-server-5.5 python python-requests supervisor && \ rm -f /etc/cron.daily/apt && \ useradd -s /bin/false pmm # ########## # # Prometheus # # ########## # - -RUN curl -s -LO https://github.com/prometheus/prometheus/releases/download/v1.0.2/prometheus-1.0.2.linux-amd64.tar.gz && \ +RUN curl -s -LO https://github.com/prometheus/prometheus/releases/download/v1.1.3/prometheus-1.1.3.linux-amd64.tar.gz && \ mkdir -p prometheus/data && \ chown -R pmm:pmm /opt/prometheus/data && \ - tar xfz prometheus-1.0.2.linux-amd64.tar.gz --strip-components=1 -C prometheus + tar xfz prometheus-1.1.3.linux-amd64.tar.gz --strip-components=1 -C prometheus && \ + rm -f prometheus-1.1.3.linux-amd64.tar.gz COPY prometheus.yml /opt/prometheus/ -# ####### # -# Grafana # -# ####### # - -RUN echo "deb https://packagecloud.io/grafana/stable/debian/ wheezy main" > /etc/apt/sources.list.d/grafana.list && \ - curl -s https://packagecloud.io/gpg.key | apt-key add - && \ - apt-get -y update && \ - apt-get -y install grafana && \ - git clone https://github.com/percona/grafana-dashboards.git && \ - git clone -b alias2instance https://github.com/roman-vynar/grafana_mongodb_dashboards.git +# ###################### # +# Grafana and dashboards # +# ###################### # COPY import-dashboards.py grafana-postinstall.sh VERSION /opt/ -RUN /opt/grafana-postinstall.sh +RUN curl -s -LO https://grafanarel.s3.amazonaws.com/builds/grafana_3.1.1-1470047149_amd64.deb && \ + dpkg -i grafana_3.1.1-1470047149_amd64.deb && \ + git clone https://github.com/percona/grafana-dashboards.git && \ + git clone -b alias2instance https://github.com/roman-vynar/grafana_mongodb_dashboards.git && \ + /opt/grafana-postinstall.sh && \ + cp /opt/VERSION /var/lib/grafana/ && \ + rm -rf grafana_3.1.1-1470047149_amd64.deb grafana-dashboards grafana_mongodb_dashboards # ###### # # Consul # # ###### # - -RUN curl -s -LO https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip && \ - unzip consul_0.6.4_linux_amd64.zip && \ +RUN curl -s -LO https://releases.hashicorp.com/consul/0.7.0/consul_0.7.0_linux_amd64.zip && \ + unzip consul_0.7.0_linux_amd64.zip && \ mkdir -p /opt/consul-data && \ - chown -R pmm:pmm /opt/consul-data + chown -R pmm:pmm /opt/consul-data && \ + rm -f consul_0.7.0_linux_amd64.zip # ##### # # Nginx # # ##### # - COPY nginx.conf nginx-ssl.conf /etc/nginx/ RUN touch /etc/nginx/.htpasswd +# ############ # +# Orchestrator # +# ############ # +COPY orchestrator.conf.json /etc/ +RUN curl -s -LO https://github.com/outbrain/orchestrator/releases/download/v1.5.6/orchestrator_1.5.6_amd64.deb && \ + dpkg -i orchestrator_1.5.6_amd64.deb && \ + curl -s -LO https://www.percona.com/downloads/TESTING/pmm/orchestrator-1.5.6-patch.tgz && \ + tar zxf orchestrator-1.5.6-patch.tgz -C /usr/local/orchestrator/ && \ + rm -f orchestrator_1.5.6_amd64.deb orchestrator-1.5.6-patch.tgz + # ########################### # # Supervisor and landing page # # ########################### # - -COPY landing-page/ /opt/landing-page/ COPY supervisord.conf /etc/supervisor/supervisord.conf COPY entrypoint.sh /opt +COPY landing-page/ /opt/landing-page/ # ####################### # # Percona Query Analytics # # ####################### # - COPY pt-archiver /usr/bin/ COPY purge-qan-data /etc/cron.daily COPY qan-install.sh /opt -ADD https://www.percona.com/downloads/TESTING/pmm/percona-qan-api-1.0.4-x86_64.tar.gz \ - https://www.percona.com/downloads/TESTING/pmm/percona-qan-app-1.0.4.tar.gz \ +ADD https://www.percona.com/downloads/TESTING/pmm/percona-qan-api-1.0.5-x86_64.tar.gz \ + https://www.percona.com/downloads/TESTING/pmm/percona-qan-app-1.0.5.tar.gz \ /opt/ RUN mkdir qan-api && \ - tar zxf percona-qan-api-1.0.4-x86_64.tar.gz --strip-components=1 -C qan-api && \ + tar zxf percona-qan-api-1.0.5-x86_64.tar.gz --strip-components=1 -C qan-api && \ mkdir qan-app && \ - tar zxf percona-qan-app-1.0.4.tar.gz --strip-components=1 -C qan-app && \ - /opt/qan-install.sh + tar zxf percona-qan-app-1.0.5.tar.gz --strip-components=1 -C qan-app && \ + /opt/qan-install.sh && \ + rm -rf percona-qan-api-1.0.5-x86_64.tar.gz percona-qan-app-1.0.5.tar.gz qan-api # ##### # # Start # diff --git a/VERSION b/VERSION index ee90284c2..90a27f9ce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +1.0.5 diff --git a/docker-compose.yml b/docker-compose.yml index 799261442..adb57727d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,9 +6,10 @@ services: image: percona/pmm-server container_name: pmm-data volumes: - - /var/lib/mysql - /opt/prometheus/data - /opt/consul-data + - /var/lib/mysql + - /var/lib/grafana entrypoint: /bin/true pmm-server: @@ -27,5 +28,7 @@ services: - METRICS_RETENTION=720h - METRICS_MEMORY=262144 - METRICS_RESOLUTION=1s + - ORCHESTRATOR_USER=orc_client_user + - ORCHESTRATOR_PASSWORD=orc_client_password # - SERVER_USER=pmm # - SERVER_PASSWORD=abc123 diff --git a/docker-push.sh b/docker-push.sh index 94eb37912..6c759764e 100755 --- a/docker-push.sh +++ b/docker-push.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -eu diff --git a/entrypoint.sh b/entrypoint.sh index 39d37eed6..69230341d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,21 @@ #!/bin/bash -replace 1s ${METRICS_RESOLUTION:-1s} -- /opt/prometheus/prometheus.yml +set -e +# Prometheus +if [[ ! "${METRICS_RESOLUTION:-1s}" =~ ^[1-5]s$ ]]; then + echo "METRICS_RESOLUTION takes only values from 1s to 5s." + exit 1 +fi +sed -i "s/1s/${METRICS_RESOLUTION:-1s}/" /opt/prometheus/prometheus.yml +sed -i "s/ENV_METRICS_RETENTION/${METRICS_RETENTION:-720h}/" /etc/supervisor/supervisord.conf +sed -i "s/ENV_METRICS_MEMORY/${METRICS_MEMORY:-262144}/" /etc/supervisor/supervisord.conf + +# Orchestrator +sed -i "s/orc_client_user/${ORCHESTRATOR_USER:-orc_client_user}/" /etc/orchestrator.conf.json +sed -i "s/orc_client_password/${ORCHESTRATOR_PASSWORD:-orc_client_password}/" /etc/orchestrator.conf.json + +# SSL if [ -e /etc/nginx/ssl/server.crt ] && [ -e /etc/nginx/ssl/server.key ]; then sed -i 's/#include nginx-ssl.conf/include nginx-ssl.conf/' /etc/nginx/nginx.conf if [ -e /etc/nginx/ssl/dhparam.pem ]; then @@ -9,12 +23,13 @@ if [ -e /etc/nginx/ssl/server.crt ] && [ -e /etc/nginx/ssl/server.key ]; then fi fi +# HTTP basic auth if [ -n "$SERVER_PASSWORD" ]; then echo "${SERVER_USER:-pmm}:$(openssl passwd -apr1 $SERVER_PASSWORD)" > /etc/nginx/.htpasswd sed -i 's/auth_basic off/auth_basic "PMM Server"/' /etc/nginx/nginx.conf - # Disable Grafana HTTP auth - sed -i '/\[auth.basic\]/ a enabled=false' /etc/grafana/grafana.ini + ENV_AUTH_BASIC="cfg:default.auth.basic.enabled=false" fi +sed -i "s/ENV_AUTH_BASIC/${ENV_AUTH_BASIC}/" /etc/supervisor/supervisord.conf supervisord -c /etc/supervisor/supervisord.conf diff --git a/grafana-postinstall.sh b/grafana-postinstall.sh index 1985fb714..90374c8b1 100755 --- a/grafana-postinstall.sh +++ b/grafana-postinstall.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -eu + service grafana-server start python /opt/import-dashboards.py diff --git a/import-dashboards.py b/import-dashboards.py index 068de237f..261775fd4 100755 --- a/import-dashboards.py +++ b/import-dashboards.py @@ -1,63 +1,110 @@ #!/usr/bin/env python -"""Grafana dashboard importer script.""" +# Grafana dashboard importer script. import json import os import requests +import shutil +import sqlite3 import sys import time -HOST = 'http://admin:admin@localhost:3000' DIRS = ['/opt/grafana-dashboards/dashboards/', '/opt/grafana_mongodb_dashboards/dashboards/'] def main(): - headers = {'Content-Type': 'application/json'} + host = 'http://127.0.0.1:3000' + api_key = 'eyJrIjoiSjZXMmM0cUpQdFp0djJRUThWMlJzNlVXQmhwRjJvVm0iLCJuIjoiUE1NIERhc2hib2FyZCBJbXBvcnQiLCJpZCI6MX0=' + db_key = '6176c9bca5590c39fc29d54b4a72e9fac5e4e8fdb75965123668d420f7b07a2d9443ad60cb8d36a1084c0fc73f3c387c0415' + headers = {'Authorization': 'Bearer %s' % (api_key,), 'Content-Type': 'application/json'} + + upgrade = False + if len(sys.argv) > 1 and sys.argv[1] == 'upgrade': + upgrade = True + + # On upgrade - check versions whether to re-import dashboards. + if upgrade: + ver1 = 'N/A' + if os.path.exists('/var/lib/grafana/VERSION'): + with open('/var/lib/grafana/VERSION', 'r') as f: + ver1 = f.read().strip() + + with open('/opt/VERSION', 'r') as f: + ver2 = f.read().strip() + + if ver1 == ver2: + print '* The dashboards are up-to-date (%s).' % (ver1,) + sys.exit(0) + + # Insert key into Grafana sqlite db. + con = sqlite3.connect('/var/lib/grafana/grafana.db') + cur = con.cursor() + cur.execute("REPLACE INTO api_key (org_id, name, key, role, created, updated) " + "VALUES (1, 'PMM Dashboard Import', '%s', 'Admin', datetime('now'), datetime('now'))" % (db_key,)) + con.commit() # Wait for Grafana to start. - for i in xrange(30): + for _ in xrange(30): try: - r = requests.get('%s/api/datasources' % (HOST,), headers=headers) + r = requests.get('%s/api/datasources' % (host,), headers=headers) except requests.exceptions.ConnectionError: print 'Waiting for Grafana to start...' time.sleep(1) else: break - # Add datasource. - data = json.dumps({'name': 'Prometheus', 'type': 'prometheus', 'url': 'http://localhost:9090/prometheus/', 'access': 'proxy', 'isDefault': True}) - r = requests.post('%s/api/datasources' % (HOST,), data=data, headers=headers) - if r.status_code != 200: - sys.exit(-1) + # Add datasource initially. + if not upgrade: + data = json.dumps({'name': 'Prometheus', 'type': 'prometheus', 'url': 'http://127.0.0.1:9090/prometheus/', 'access': 'proxy', 'isDefault': True}) + r = requests.post('%s/api/datasources' % (host,), data=data, headers=headers) + print r.status_code, r.content + if r.status_code != 200: + sys.exit(-1) - print r.status_code, r.content - - # Import dashboards. + # Import dashboards with overwrite. files = [] for d in DIRS: for f in os.listdir(d): if not f.endswith('.json'): - continue + continue files.append(d + f) - for file in files: - print file - f = open(file, 'r') + for file_ in files: + print file_ + f = open(file_, 'r') dash = json.load(f) f.close() - # Set time options. + # Set time range and refresh options. dash['time']['from'] = 'now-1h' dash['time']['to'] = 'now' dash['refresh'] = '1m' data = json.dumps({'dashboard': dash, 'overwrite': True}) - r = requests.post('%s/api/dashboards/db' % (HOST,), data=data, headers=headers) + r = requests.post('%s/api/dashboards/db' % (host,), data=data, headers=headers) if r.status_code != 200: print r.status_code, r.content - sys.exit(-1) + sys.exit(-1) + + # Set home dashboard. + if not upgrade: + cur.execute("INSERT INTO star (user_id, dashboard_id) " + "SELECT 1, id from dashboard WHERE slug='cross-server-graphs'") + cur.execute("INSERT INTO preferences (org_id, user_id, version, home_dashboard_id, timezone, theme, created, updated) " + "SELECT 1, 1, 0, id, '', '', datetime('now'), datetime('now') from dashboard WHERE slug='cross-server-graphs'") + + # Delete key. + cur.execute("DELETE FROM api_key WHERE key='%s'" % (db_key,)) + + con.commit() + con.close() + + # On upgrade - update VERSION file. + if upgrade: + shutil.copyfile('/opt/VERSION', '/var/lib/grafana/VERSION') + print '* Dashboards upgraded successfully from version %s to %s.' % (ver1, ver2) if __name__ == '__main__': diff --git a/landing-page/index.html b/landing-page/index.html index b8d025d2c..4f8bbee96 100644 --- a/landing-page/index.html +++ b/landing-page/index.html @@ -1,7 +1,7 @@ - + @@ -11,13 +11,13 @@ - + Percona Monitoring and Management - - + +
- + - +

Percona Monitoring and Management

Percona Monitoring and Management (PMM) is a free and open-source solution for managing and monitoring performance on MySQL and MongoDB, and provides time-based analysis of performance to ensure that your data works as efficiently as possible.

+
+
+
+

Review the documentation

+

Download User Manual

+
- -
+ +
- +
Percona Monitoring and Management Tool
- +
-
-
QUERY ANALYTICS + +
+
Query
Analytics
-
-
METRICS MONITOR +
+
Metrics
Monitor
-
DOCUMENTATION +
MySQL Replication
Topology Manager
- +

Tell us what you think

-

We love feedback. Click below to send us an email and we'll quickly get back to you.

- SEND FEEDBACK EMAIL +

We welcome your feedback, questions and comments. Please visit our PMM forums for questions and discussions.

- + diff --git a/nginx.conf b/nginx.conf index d8ae35ad0..bc98a12ea 100644 --- a/nginx.conf +++ b/nginx.conf @@ -37,23 +37,23 @@ http { # Grafana rewrite ^/graph$ /graph/; location /graph { - proxy_pass http://localhost:3000; + proxy_pass http://127.0.0.1:3000; rewrite ^/graph/(.*) /$1 break; } # Prometheus location /prometheus { - proxy_pass http://localhost:9090; + proxy_pass http://127.0.0.1:9090; } # Consul UI location /consul/ { - proxy_pass http://localhost:8500/ui/; + proxy_pass http://127.0.0.1:8500/ui/; } # Consul API location /v1/ { - proxy_pass http://localhost:8500/v1/; + proxy_pass http://127.0.0.1:8500/v1/; add_header X-Remote-IP $remote_addr; } @@ -65,7 +65,7 @@ http { # QAN API rewrite ^/qan-api$ /qan-api/; location /qan-api { - proxy_pass http://localhost:9001; + proxy_pass http://127.0.0.1:9001; rewrite ^/qan-api/(.*) /$1 break; proxy_set_header Host $http_host; proxy_http_version 1.1; @@ -73,6 +73,11 @@ http { proxy_set_header Connection $connection_upgrade; proxy_read_timeout 86400; } + + # Orchestrator + location /orchestrator { + proxy_pass http://127.0.0.1:4000; + } } } diff --git a/orchestrator.conf.json b/orchestrator.conf.json new file mode 100644 index 000000000..fc21f91c5 --- /dev/null +++ b/orchestrator.conf.json @@ -0,0 +1,145 @@ +{ + "Debug": false, + "EnableSyslog": false, + "ListenAddress": ":4000", + "MySQLTopologyUser": "orc_client_user", + "MySQLTopologyPassword": "orc_client_password", + "MySQLTopologyCredentialsConfigFile": "", + "MySQLTopologySSLPrivateKeyFile": "", + "MySQLTopologySSLCertFile": "", + "MySQLTopologySSLCAFile": "", + "MySQLTopologySSLSkipVerify": true, + "MySQLTopologyUseMutualTLS": false, + "MySQLTopologyMaxPoolConnections": 3, + "DatabaselessMode__experimental": false, + "MySQLOrchestratorHost": "127.0.0.1", + "MySQLOrchestratorPort": 3306, + "MySQLOrchestratorDatabase": "orchestrator", + "MySQLOrchestratorUser": "orchestrator", + "MySQLOrchestratorPassword": "orchestrator", + "MySQLOrchestratorCredentialsConfigFile": "", + "MySQLOrchestratorSSLPrivateKeyFile": "", + "MySQLOrchestratorSSLCertFile": "", + "MySQLOrchestratorSSLCAFile": "", + "MySQLOrchestratorSSLSkipVerify": true, + "MySQLOrchestratorUseMutualTLS": false, + "MySQLConnectTimeoutSeconds": 1, + "DefaultInstancePort": 3306, + "SlaveStartPostWaitMilliseconds": 1000, + "DiscoverByShowSlaveHosts": true, + "InstancePollSeconds": 5, + "ReadLongRunningQueries": true, + "UnseenInstanceForgetHours": 240, + "SnapshotTopologiesIntervalHours": 0, + "InstanceBulkOperationsWaitTimeoutSeconds": 10, + "ActiveNodeExpireSeconds": 5, + "HostnameResolveMethod": "none", + "MySQLHostnameResolveMethod": "none", + "SkipBinlogServerUnresolveCheck": true, + "ExpiryHostnameResolvesMinutes": 60, + "RejectHostnameResolvePattern": "", + "ReasonableReplicationLagSeconds": 10, + "ProblemIgnoreHostnameFilters": [], + "VerifyReplicationFilters": false, + "MaintenanceOwner": "orchestrator", + "ReasonableMaintenanceReplicationLagSeconds": 20, + "MaintenanceExpireMinutes": 10, + "MaintenancePurgeDays": 365, + "CandidateInstanceExpireMinutes": 60, + "AuditLogFile": "", + "AuditToSyslog": false, + "AuditPageSize": 20, + "AuditPurgeDays": 365, + "RemoveTextFromHostnameDisplay": ".mydomain.com:3306", + "ReadOnly": false, + "AuthenticationMethod": "", + "HTTPAuthUser": "", + "HTTPAuthPassword": "", + "AuthUserHeader": "", + "PowerAuthUsers": [ + "*" + ], + "ClusterNameToAlias": { + "127.0.0.1": "test suite" + }, + "SlaveLagQuery": "", + "DetectClusterAliasQuery": "SELECT SUBSTRING_INDEX(@@hostname, '.', 1)", + "DetectClusterDomainQuery": "", + "DetectInstanceAliasQuery": "", + "DetectPromotionRuleQuery": "", + "DataCenterPattern": "[.]([^.]+)[.][^.]+[.]mydomain[.]com", + "PhysicalEnvironmentPattern": "[.]([^.]+[.][^.]+)[.]mydomain[.]com", + "PromotionIgnoreHostnameFilters": [], + "DetectSemiSyncEnforcedQuery": "", + "ServeAgentsHttp": false, + "AgentsServerPort": ":3001", + "AgentsUseSSL": false, + "AgentsUseMutualTLS": false, + "AgentSSLSkipVerify": false, + "AgentSSLPrivateKeyFile": "", + "AgentSSLCertFile": "", + "AgentSSLCAFile": "", + "AgentSSLValidOUs": [], + "UseSSL": false, + "UseMutualTLS": false, + "SSLSkipVerify": false, + "SSLPrivateKeyFile": "", + "SSLCertFile": "", + "SSLCAFile": "", + "SSLValidOUs": [], + "URLPrefix": "/orchestrator", + "StatusEndpoint": "/orchestrator/api/status", + "StatusSimpleHealth": true, + "StatusOUVerify": false, + "HttpTimeoutSeconds": 60, + "AgentPollMinutes": 60, + "AgentAutoDiscover": false, + "UnseenAgentForgetHours": 6, + "StaleSeedFailMinutes": 60, + "SeedAcceptableBytesDiff": 8192, + "PseudoGTIDPattern": "", + "PseudoGTIDPatternIsFixedSubstring": false, + "PseudoGTIDMonotonicHint": "asc:", + "DetectPseudoGTIDQuery": "", + "PseudoGTIDCoordinatesHistoryHeuristicMinutes": 2, + "BinlogEventsChunkSize": 10000, + "BufferBinlogEvents": true, + "SkipBinlogEventsContaining": [], + "ReduceReplicationAnalysisCount": true, + "FailureDetectionPeriodBlockMinutes": 60, + "RecoveryPollSeconds": 10, + "RecoveryPeriodBlockSeconds": 3600, + "RecoveryIgnoreHostnameFilters": [], + "RecoverMasterClusterFilters": [ + "_master_pattern_" + ], + "RecoverIntermediateMasterClusterFilters": [ + "_intermediate_master_pattern_" + ], + "OnFailureDetectionProcesses": [ + "echo 'Detected {failureType} on {failureCluster}. Affected replicas: {countSlaves}' >> /tmp/recovery.log" + ], + "PreFailoverProcesses": [ + "echo 'Will recover from {failureType} on {failureCluster}' >> /tmp/recovery.log" + ], + "PostFailoverProcesses": [ + "echo '(for all types) Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "PostUnsuccessfulFailoverProcesses": [], + "PostMasterFailoverProcesses": [ + "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Promoted: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "PostIntermediateMasterFailoverProcesses": [ + "echo 'Recovered from {failureType} on {failureCluster}. Failed: {failedHost}:{failedPort}; Successor: {successorHost}:{successorPort}' >> /tmp/recovery.log" + ], + "CoMasterRecoveryMustPromoteOtherCoMaster": true, + "DetachLostSlavesAfterMasterFailover": true, + "ApplyMySQLPromotionAfterMasterFailover": false, + "MasterFailoverDetachSlaveMasterHost": false, + "MasterFailoverLostInstancesDowntimeMinutes": 0, + "PostponeSlaveRecoveryOnLagMinutes": 0, + "OSCIgnoreHostnameFilters": [], + "GraphiteAddr": "", + "GraphitePath": "", + "GraphiteConvertHostnameDotsToUnderscores": true +} diff --git a/prometheus.yml b/prometheus.yml index e01dcb61c..36534d1d1 100644 --- a/prometheus.yml +++ b/prometheus.yml @@ -15,6 +15,7 @@ scrape_configs: - job_name: linux consul_sd_configs: - server: 'localhost:8500' + datacenter: dc1 services: ['linux:metrics'] relabel_configs: @@ -24,9 +25,23 @@ scrape_configs: replacement: '$1' + - job_name: proxysql + consul_sd_configs: + - server: 'localhost:8500' + datacenter: dc1 + services: ['proxysql:metrics'] + + relabel_configs: + - source_labels: ['__meta_consul_tags'] + regex: '.*,alias_([-\w:\.]+),.*' + target_label: 'instance' + replacement: '$1' + + - job_name: mongodb consul_sd_configs: - server: 'localhost:8500' + datacenter: dc1 services: ['mongodb:metrics'] relabel_configs: @@ -53,6 +68,7 @@ scrape_configs: consul_sd_configs: - server: 'localhost:8500' + datacenter: dc1 services: ['mysql:metrics'] relabel_configs: @@ -71,6 +87,7 @@ scrape_configs: consul_sd_configs: - server: 'localhost:8500' + datacenter: dc1 services: ['mysql:metrics'] relabel_configs: @@ -89,6 +106,7 @@ scrape_configs: consul_sd_configs: - server: 'localhost:8500' + datacenter: dc1 services: ['mysql:metrics'] relabel_configs: diff --git a/qan-install.sh b/qan-install.sh index 22ca00fe2..a55a4f4f5 100755 --- a/qan-install.sh +++ b/qan-install.sh @@ -1,9 +1,13 @@ -#!/usr/bin/env bash +#!/bin/bash set -eu service mysql start +# Create Orchestrator db. +mysql -vv -e "CREATE DATABASE IF NOT EXISTS orchestrator; GRANT ALL PRIVILEGES ON orchestrator.* TO 'orchestrator'@'localhost' IDENTIFIED BY 'orchestrator'" + +# Install QAN API. # START=no SYSINT=no because Supervisor starts and manages these processes. cd /opt/qan-api START="no" SYSINT="no" ./install diff --git a/supervisord.conf b/supervisord.conf index 932c070b3..c182b58ef 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -22,8 +22,8 @@ password = dummy priority = 1 user = mysql command = /usr/bin/mysqld_safe -stdout_logfile = /var/log/mysqld.log -stderr_logfile = /var/log/mysqld.log +stdout_logfile = /var/log/mysql.log +stderr_logfile = /var/log/mysql.log autorestart = true [program:consul] @@ -37,7 +37,7 @@ autorestart = true [program:grafana] priority = 3 user = grafana -command = /usr/sbin/grafana-server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini cfg:default.paths.data=/var/lib/grafana cfg:default.paths.logs=/var/log/grafana cfg:default.server.root_url="%%(protocol)s://%%(domain)s:%%(http_port)s/graph" +command = /usr/sbin/grafana-server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini cfg:default.paths.data=/var/lib/grafana cfg:default.paths.logs=/var/log/grafana cfg:default.paths.plugins=/var/lib/grafana/plugins cfg:default.server.root_url="%%(protocol)s://%%(domain)s:%%(http_port)s/graph" ENV_AUTH_BASIC stdout_logfile = /var/log/grafana.log stderr_logfile = /var/log/grafana.log autorestart = true @@ -64,14 +64,44 @@ environment = BG=no,BASE_PATH=/qan-api command = bash -c "sleep 5 && /usr/local/percona/qan-api/start" stdout_logfile = /var/log/qan-api.log stderr_logfile = /var/log/qan-api.log +startretries = 60 autorestart = true [program:prometheus] priority = 7 user = pmm -# Sleep to wait for consul to start up. -command = bash -c "sleep 5 && /opt/prometheus/prometheus -config.file=/opt/prometheus/prometheus.yml -storage.local.path=/opt/prometheus/data -web.listen-address=:9090 -storage.local.retention=${METRICS_RETENTION:-720h} -storage.local.memory-chunks=${METRICS_MEMORY:-262144} -web.console.libraries=/opt/prometheus/console_libraries -web.console.templates=/opt/prometheus/consoles -web.external-url=http://localhost:9090/prometheus/" +command = /opt/prometheus/prometheus -config.file=/opt/prometheus/prometheus.yml -storage.local.path=/opt/prometheus/data -web.listen-address=:9090 -storage.local.retention=ENV_METRICS_RETENTION -storage.local.memory-chunks=ENV_METRICS_MEMORY -storage.local.chunk-encoding-version=2 -web.console.libraries=/opt/prometheus/console_libraries -web.console.templates=/opt/prometheus/consoles -web.external-url=http://localhost:9090/prometheus/ stdout_logfile = /var/log/prometheus.log stderr_logfile = /var/log/prometheus.log autorestart = true +# This is here to support data containers of v1.0.4. +[program:createdb] +priority = 8 +# Sleep to wait for mysql to start up. +command = bash -c "sleep 5 && mysql -vv -e \"CREATE DATABASE IF NOT EXISTS orchestrator; GRANT ALL PRIVILEGES ON orchestrator.* TO 'orchestrator'@'localhost' IDENTIFIED BY 'orchestrator'\"" +stdout_logfile = /var/log/createdb.log +stderr_logfile = /var/log/createdb.log +startretries = 60 +autorestart = unexpected + +[program:orchestrator] +priority = 9 +user = pmm +directory = /usr/local/orchestrator +# Sleep to wait for mysql to start up. +command = bash -c "sleep 7 && /usr/local/orchestrator/orchestrator http" +stdout_logfile = /var/log/orchestrator.log +stderr_logfile = /var/log/orchestrator.log +startretries = 60 +autorestart = true + +[program:dashboard-upgrade] +priority = 10 +command = python /opt/import-dashboards.py upgrade +stdout_logfile = /var/log/dashboard-upgrade.log +stderr_logfile = /var/log/dashboard-upgrade.log +startsecs = 0 +startretries = 60 +autorestart = unexpected +