1.0.5 release.

* New tool: Orchestrator - a MySQL replication topology management and visualization tool. Available by /orchestrator URL. * Added ProxySQL metrics and dashboard. * Prometheus 1.1.3. * Consul 0.7.0. * Grafana data can be stored on data container to preserve your custom dashboards and settings. The predefined dashboards are still the subject of upgrade on new versions. * Changed metric storage encoding to achieve less disk space usage by 50-70%. * Re-use MySQL instance with saved data on Query Analytics when re-added through pmm-admin with the same name. * Various dashboard improvements. * Fixed rare issue when nginx tries to use IPv6 for localhost connections. * Improvements and fixes to Query Analytics.
adivinho · Oct 13, 2016 · 8fda1fb · 8fda1fb
1 parent bce944d
commit 8fda1fb
Show file tree

Hide file tree

Showing 14 changed files with 387 additions and 100 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,18 @@
 Percona Monitoring and Management (PMM) Server
 
+v1.0.5 released 2016-10-14
+
+* New tool: Orchestrator - a MySQL replication topology management and visualization tool. Available by /orchestrator URL.
+* Added ProxySQL metrics and dashboard.
+* Prometheus 1.1.3.
+* Consul 0.7.0.
+* Grafana data can be stored on data container to preserve your custom dashboards and settings. The predefined dashboards are still the subject of upgrade on new versions.
+* Changed metric storage encoding to achieve less disk space usage by 50-70%.
+* Re-use MySQL instance with saved data on Query Analytics when re-added through pmm-admin with the same name.
+* Various dashboard improvements.
+* Fixed rare issue when nginx tries to use IPv6 for localhost connections.
+* Improvements and fixes to Query Analytics.
+
 v1.0.4 released 2016-09-13
 
 * Grafana 3.1.1.
@@ -17,7 +30,7 @@ v1.0.4 released 2016-09-13
 
 v1.0.3 released 2016-08-05
 
-* Fixed math for query metrics on QAN App.
+* Fixed math for query metrics on Query Analytics.
 
 v1.0.2 released 2016-07-28
 
@@ -42,7 +55,7 @@ v1.0.1 released 2016-06-09
 * Added MongoDB dashboards.
 * Replaced prom-config-api with Consul 0.6.4.
 * Eliminated most of the ports for PMM server container (now only two: 9001 and configurable 80).
-* Added "Server Summary" with aggregated query metrics to QAN app.
+* Added "Server Summary" with aggregated query metrics to Query Analytics.
 * MySQL dashboard updates, added "MySQL InnoDB Metrics Advanced" dashboard.
 * Added options to configure Prometheus memory and retention.
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,88 +1,86 @@
-FROM ubuntu:latest
+FROM ubuntu:14.04
 
 EXPOSE 80 443
 
 WORKDIR /opt
 
-# ########################### #
-# MySQL and other system pkgs #
-# ########################### #
-
-RUN apt-get -y update && apt-get install -y \
-	apt-transport-https \
-	curl \
-	git \
-	unzip \
-	nginx \
-	mysql-server \
-	python \
-	python-requests \
-	supervisor && \
+# ############### #
+# System packages #
+# ############### #
+RUN apt-get -y update && \
+	apt-get install -y apt-transport-https curl git unzip nginx mysql-server-5.5 python python-requests supervisor && \
 	rm -f /etc/cron.daily/apt && \
 	useradd -s /bin/false pmm
 
 # ########## #
 # Prometheus #
 # ########## #
-
-RUN curl -s -LO https://github.com/prometheus/prometheus/releases/download/v1.0.2/prometheus-1.0.2.linux-amd64.tar.gz && \
+RUN curl -s -LO https://github.com/prometheus/prometheus/releases/download/v1.1.3/prometheus-1.1.3.linux-amd64.tar.gz && \
 	mkdir -p prometheus/data && \
 	chown -R pmm:pmm /opt/prometheus/data && \
-	tar xfz prometheus-1.0.2.linux-amd64.tar.gz --strip-components=1 -C prometheus
+	tar xfz prometheus-1.1.3.linux-amd64.tar.gz --strip-components=1 -C prometheus && \
+	rm -f prometheus-1.1.3.linux-amd64.tar.gz
 COPY prometheus.yml /opt/prometheus/
 
-# ####### #
-# Grafana #
-# ####### #
-
-RUN echo "deb https://packagecloud.io/grafana/stable/debian/ wheezy main" > /etc/apt/sources.list.d/grafana.list && \
-	curl -s https://packagecloud.io/gpg.key | apt-key add - && \
-	apt-get -y update && \
-	apt-get -y install grafana && \
-	git clone https://github.com/percona/grafana-dashboards.git && \
-	git clone -b alias2instance https://github.com/roman-vynar/grafana_mongodb_dashboards.git
+# ###################### #
+# Grafana and dashboards #
+# ###################### #
 COPY import-dashboards.py grafana-postinstall.sh VERSION /opt/
-RUN /opt/grafana-postinstall.sh
+RUN curl -s -LO https://grafanarel.s3.amazonaws.com/builds/grafana_3.1.1-1470047149_amd64.deb && \
+	dpkg -i grafana_3.1.1-1470047149_amd64.deb && \
+	git clone https://github.com/percona/grafana-dashboards.git && \
+	git clone -b alias2instance https://github.com/roman-vynar/grafana_mongodb_dashboards.git && \
+	/opt/grafana-postinstall.sh && \
+	cp /opt/VERSION /var/lib/grafana/ && \
+	rm -rf grafana_3.1.1-1470047149_amd64.deb grafana-dashboards grafana_mongodb_dashboards
 
 # ###### #
 # Consul #
 # ###### #
-
-RUN curl -s -LO https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip && \
-	unzip consul_0.6.4_linux_amd64.zip && \
+RUN curl -s -LO https://releases.hashicorp.com/consul/0.7.0/consul_0.7.0_linux_amd64.zip && \
+	unzip consul_0.7.0_linux_amd64.zip && \
 	mkdir -p /opt/consul-data && \
-	chown -R pmm:pmm /opt/consul-data
+	chown -R pmm:pmm /opt/consul-data && \
+	rm -f consul_0.7.0_linux_amd64.zip
 
 # ##### #
 # Nginx #
 # ##### #
-
 COPY nginx.conf nginx-ssl.conf /etc/nginx/
 RUN touch /etc/nginx/.htpasswd
 
+# ############ #
+# Orchestrator #
+# ############ #
+COPY orchestrator.conf.json /etc/
+RUN curl -s -LO https://github.com/outbrain/orchestrator/releases/download/v1.5.6/orchestrator_1.5.6_amd64.deb && \
+	dpkg -i orchestrator_1.5.6_amd64.deb && \
+	curl -s -LO https://www.percona.com/downloads/TESTING/pmm/orchestrator-1.5.6-patch.tgz && \
+	tar zxf orchestrator-1.5.6-patch.tgz -C /usr/local/orchestrator/ && \
+	rm -f orchestrator_1.5.6_amd64.deb orchestrator-1.5.6-patch.tgz
+
 # ########################### #
 # Supervisor and landing page # 
 # ########################### #
-
-COPY landing-page/ /opt/landing-page/
 COPY supervisord.conf /etc/supervisor/supervisord.conf
 COPY entrypoint.sh /opt
+COPY landing-page/ /opt/landing-page/
 
 # ####################### #
 # Percona Query Analytics #
 # ####################### #
-
 COPY pt-archiver /usr/bin/
 COPY purge-qan-data /etc/cron.daily
 COPY qan-install.sh /opt
-ADD https://www.percona.com/downloads/TESTING/pmm/percona-qan-api-1.0.4-x86_64.tar.gz \
-    https://www.percona.com/downloads/TESTING/pmm/percona-qan-app-1.0.4.tar.gz \
+ADD https://www.percona.com/downloads/TESTING/pmm/percona-qan-api-1.0.5-x86_64.tar.gz \
+    https://www.percona.com/downloads/TESTING/pmm/percona-qan-app-1.0.5.tar.gz \
     /opt/
 RUN mkdir qan-api && \
-        tar zxf percona-qan-api-1.0.4-x86_64.tar.gz --strip-components=1 -C qan-api && \
+        tar zxf percona-qan-api-1.0.5-x86_64.tar.gz --strip-components=1 -C qan-api && \
         mkdir qan-app && \
-        tar zxf percona-qan-app-1.0.4.tar.gz --strip-components=1 -C qan-app && \
-	/opt/qan-install.sh
+        tar zxf percona-qan-app-1.0.5.tar.gz --strip-components=1 -C qan-app && \
+	/opt/qan-install.sh && \
+	rm -rf percona-qan-api-1.0.5-x86_64.tar.gz percona-qan-app-1.0.5.tar.gz qan-api
 
 # ##### #
 # Start #

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.0.4
+1.0.5
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -6,9 +6,10 @@ services:
     image: percona/pmm-server
     container_name: pmm-data
     volumes:
-      - /var/lib/mysql
       - /opt/prometheus/data
       - /opt/consul-data
+      - /var/lib/mysql
+      - /var/lib/grafana
     entrypoint: /bin/true
 
   pmm-server:
@@ -27,5 +28,7 @@ services:
       - METRICS_RETENTION=720h
       - METRICS_MEMORY=262144
       - METRICS_RESOLUTION=1s
+      - ORCHESTRATOR_USER=orc_client_user
+      - ORCHESTRATOR_PASSWORD=orc_client_password
 #      - SERVER_USER=pmm
 #      - SERVER_PASSWORD=abc123
diff --git a/docker-push.sh b/docker-push.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -eu
 

diff --git a/entrypoint.sh b/entrypoint.sh
@@ -1,20 +1,35 @@
 #!/bin/bash
 
-replace 1s ${METRICS_RESOLUTION:-1s} -- /opt/prometheus/prometheus.yml
+set -e
 
+# Prometheus
+if [[ ! "${METRICS_RESOLUTION:-1s}" =~ ^[1-5]s$ ]]; then
+    echo "METRICS_RESOLUTION takes only values from 1s to 5s."
+    exit 1
+fi
+sed -i "s/1s/${METRICS_RESOLUTION:-1s}/" /opt/prometheus/prometheus.yml
+sed -i "s/ENV_METRICS_RETENTION/${METRICS_RETENTION:-720h}/" /etc/supervisor/supervisord.conf
+sed -i "s/ENV_METRICS_MEMORY/${METRICS_MEMORY:-262144}/" /etc/supervisor/supervisord.conf
+
+# Orchestrator
+sed -i "s/orc_client_user/${ORCHESTRATOR_USER:-orc_client_user}/" /etc/orchestrator.conf.json
+sed -i "s/orc_client_password/${ORCHESTRATOR_PASSWORD:-orc_client_password}/" /etc/orchestrator.conf.json
+
+# SSL
 if [ -e /etc/nginx/ssl/server.crt ] && [ -e /etc/nginx/ssl/server.key ]; then
     sed -i 's/#include nginx-ssl.conf/include nginx-ssl.conf/' /etc/nginx/nginx.conf
     if [ -e /etc/nginx/ssl/dhparam.pem ]; then
         sed -i 's/#ssl_dhparam/ssl_dhparam/' /etc/nginx/nginx-ssl.conf
     fi
 fi
 
+# HTTP basic auth
 if [ -n "$SERVER_PASSWORD" ]; then
     echo "${SERVER_USER:-pmm}:$(openssl passwd -apr1 $SERVER_PASSWORD)" > /etc/nginx/.htpasswd
     sed -i 's/auth_basic off/auth_basic "PMM Server"/' /etc/nginx/nginx.conf
 
-    # Disable Grafana HTTP auth
-    sed -i '/\[auth.basic\]/ a enabled=false' /etc/grafana/grafana.ini
+    ENV_AUTH_BASIC="cfg:default.auth.basic.enabled=false"
 fi
+sed -i "s/ENV_AUTH_BASIC/${ENV_AUTH_BASIC}/" /etc/supervisor/supervisord.conf
 
 supervisord -c /etc/supervisor/supervisord.conf
diff --git a/grafana-postinstall.sh b/grafana-postinstall.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -eu
+
 service grafana-server start
 
 python /opt/import-dashboards.py

diff --git a/import-dashboards.py b/import-dashboards.py
@@ -1,63 +1,110 @@
 #!/usr/bin/env python
 
-"""Grafana dashboard importer script."""
+# Grafana dashboard importer script.
 
 import json
 import os
 import requests
+import shutil
+import sqlite3
 import sys
 import time
 
-HOST = 'http://admin:admin@localhost:3000'
 DIRS = ['/opt/grafana-dashboards/dashboards/', '/opt/grafana_mongodb_dashboards/dashboards/']
 
 
 def main():
-    headers = {'Content-Type': 'application/json'}
+    host = 'http://127.0.0.1:3000'
+    api_key = 'eyJrIjoiSjZXMmM0cUpQdFp0djJRUThWMlJzNlVXQmhwRjJvVm0iLCJuIjoiUE1NIERhc2hib2FyZCBJbXBvcnQiLCJpZCI6MX0='
+    db_key = '6176c9bca5590c39fc29d54b4a72e9fac5e4e8fdb75965123668d420f7b07a2d9443ad60cb8d36a1084c0fc73f3c387c0415'
+    headers = {'Authorization': 'Bearer %s' % (api_key,), 'Content-Type': 'application/json'}
+
+    upgrade = False
+    if len(sys.argv) > 1 and sys.argv[1] == 'upgrade':
+        upgrade = True
+
+    # On upgrade - check versions whether to re-import dashboards.
+    if upgrade:
+        ver1 = 'N/A'
+        if os.path.exists('/var/lib/grafana/VERSION'):
+            with open('/var/lib/grafana/VERSION', 'r') as f:
+                ver1 = f.read().strip()
+
+        with open('/opt/VERSION', 'r') as f:
+            ver2 = f.read().strip()
+
+        if ver1 == ver2:
+            print '* The dashboards are up-to-date (%s).' % (ver1,)
+            sys.exit(0)
+
+    # Insert key into Grafana sqlite db.
+    con = sqlite3.connect('/var/lib/grafana/grafana.db')
+    cur = con.cursor()
+    cur.execute("REPLACE INTO api_key (org_id, name, key, role, created, updated) "
+                "VALUES (1, 'PMM Dashboard Import', '%s', 'Admin', datetime('now'), datetime('now'))" % (db_key,))
+    con.commit()
 
     # Wait for Grafana to start.
-    for i in xrange(30): 
+    for _ in xrange(30):
         try:
-            r = requests.get('%s/api/datasources' % (HOST,), headers=headers)
+            r = requests.get('%s/api/datasources' % (host,), headers=headers)
         except requests.exceptions.ConnectionError:
             print 'Waiting for Grafana to start...'
             time.sleep(1)
         else:
             break
 
-    # Add datasource.
-    data = json.dumps({'name': 'Prometheus', 'type': 'prometheus', 'url': 'http://localhost:9090/prometheus/', 'access': 'proxy', 'isDefault': True})
-    r = requests.post('%s/api/datasources' % (HOST,), data=data, headers=headers)
-    if r.status_code != 200:
-        sys.exit(-1)
+    # Add datasource initially.
+    if not upgrade:
+        data = json.dumps({'name': 'Prometheus', 'type': 'prometheus', 'url': 'http://127.0.0.1:9090/prometheus/', 'access': 'proxy', 'isDefault': True})
+        r = requests.post('%s/api/datasources' % (host,), data=data, headers=headers)
+        print r.status_code, r.content
+        if r.status_code != 200:
+            sys.exit(-1)
 
-    print r.status_code, r.content
-
-    # Import dashboards.
+    # Import dashboards with overwrite.
     files = []
     for d in DIRS:
         for f in os.listdir(d):
             if not f.endswith('.json'):
-                 continue
+                continue
 
             files.append(d + f)
 
-    for file in files:
-        print file
-        f = open(file, 'r')
+    for file_ in files:
+        print file_
+        f = open(file_, 'r')
         dash = json.load(f)
         f.close()
 
-        # Set time options.
+        # Set time range and refresh options.
         dash['time']['from'] = 'now-1h'
         dash['time']['to'] = 'now'
         dash['refresh'] = '1m'
 
         data = json.dumps({'dashboard': dash, 'overwrite': True})
-        r = requests.post('%s/api/dashboards/db' % (HOST,), data=data, headers=headers)
+        r = requests.post('%s/api/dashboards/db' % (host,), data=data, headers=headers)
         if r.status_code != 200:
             print r.status_code, r.content
-            sys.exit(-1) 
+            sys.exit(-1)
+
+    # Set home dashboard.
+    if not upgrade:
+        cur.execute("INSERT INTO star (user_id, dashboard_id) "
+                    "SELECT 1, id from dashboard WHERE slug='cross-server-graphs'")
+        cur.execute("INSERT INTO preferences (org_id, user_id, version, home_dashboard_id, timezone, theme, created, updated) "
+                    "SELECT 1, 1, 0, id, '', '', datetime('now'), datetime('now') from dashboard WHERE slug='cross-server-graphs'")
+
+    # Delete key.
+    cur.execute("DELETE FROM api_key WHERE key='%s'" % (db_key,))
+
+    con.commit()
+    con.close()
+
+    # On upgrade - update VERSION file.
+    if upgrade:
+        shutil.copyfile('/opt/VERSION', '/var/lib/grafana/VERSION')
+        print '* Dashboards upgraded successfully from version %s to %s.' % (ver1, ver2)
 
 
 if __name__ == '__main__':