Merge pull request percona#2 from percona/consul

New version: switched to Consul, Nginx, added MongoDB

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:latest
 
-EXPOSE 3000 9090 9001-9003 80
+EXPOSE 80 9001
 
 WORKDIR /opt
 
@@ -12,6 +12,8 @@ RUN apt-get -y update && apt-get install -y \
 	apt-transport-https \
 	curl \
 	git \
+	unzip \
+	nginx \
 	mysql-server \
 	python \
 	supervisor
@@ -20,9 +22,9 @@ RUN apt-get -y update && apt-get install -y \
 # Prometheus #
 # ########## #
 
-ADD https://github.com/prometheus/prometheus/releases/download/0.17.0/prometheus-0.17.0.linux-amd64.tar.gz /opt/
+ADD https://github.com/prometheus/prometheus/releases/download/0.19.2/prometheus-0.19.2.linux-amd64.tar.gz /opt/
 RUN mkdir prometheus && \
-	tar xfz prometheus-0.17.0.linux-amd64.tar.gz --strip-components=1 -C prometheus
+	tar xfz prometheus-0.19.2.linux-amd64.tar.gz --strip-components=1 -C prometheus
 COPY prometheus.yml /opt/prometheus/
 
 # ####### #
@@ -35,39 +37,47 @@ RUN echo "deb https://packagecloud.io/grafana/stable/debian/ wheezy main" > /etc
 	apt-get -y install grafana
 RUN git clone https://github.com/percona/grafana-dashboards.git && \
 	mkdir /var/lib/grafana/dashboards && \
-	cp grafana-dashboards/dashboards/* /var/lib/grafana/dashboards/ && \
-	rm -f /var/lib/grafana/dashboards/*InfluxDB*
+	cp grafana-dashboards/dashboards/* /var/lib/grafana/dashboards/
+RUN git clone https://github.com/Percona-Lab/grafana_mongodb_dashboards.git && \
+	cp grafana_mongodb_dashboards/dashboards/* /var/lib/grafana/dashboards/
 COPY grafana.ini /etc/grafana/grafana.ini
 COPY add-grafana-datasource.sh /opt
 RUN chgrp grafana /etc/grafana/grafana.ini && \
-	sed -i 's/step_input:""/step_input:c.target.step/; s/ HH:MM/ HH:mm/; s/,function(c)/,"templateSrv",function(c,g)/; s/expr:c.target.expr/expr:g.replace(c.target.expr,c.panel.scopedVars)/' /usr/share/grafana/public/app/plugins/datasource/prometheus/query_ctrl.js && \
-	sed -i 's/h=a.interval/h=g.replace(a.interval, c.scopedVars)/' /usr/share/grafana/public/app/plugins/datasource/prometheus/datasource.js && \
-	/opt/add-grafana-datasource.sh
+	/opt/add-grafana-datasource.sh && \
+	sed -i 's/expr=\(.\)\.replace(\(.\)\.expr,\(.\)\.scopedVars\(.*\)var \(.\)=\(.\)\.interval/expr=\1.replace(\2.expr,\3.scopedVars\4var \5=\1.replace(\6.interval, \3.scopedVars)/' /usr/share/grafana/public/app/plugins/datasource/prometheus/datasource.js && \
+	sed -i 's/,range_input/.replace(\/"{\/g,"\\"").replace(\/}"\/g,"\\""),range_input/; s/step_input:""/step_input:this.target.step/' /usr/share/grafana/public/app/plugins/datasource/prometheus/query_ctrl.js
 
 # ####################### #
 # Percona Query Analytics #
 # ####################### #
 
-ADD https://www.percona.com/downloads/TESTING/pmm/percona-qan-api-1.0.0-x86_64.tar.gz \
-    https://www.percona.com/downloads/TESTING/pmm/percona-qan-app-1.0.0.tar.gz \
+ADD https://www.percona.com/downloads/TESTING/pmm/percona-qan-api.tar.gz \
+    https://www.percona.com/downloads/TESTING/pmm/percona-qan-app.tar.gz \
     /opt/
 RUN mkdir qan-api && \
-	tar zxf percona-qan-api-1.0.0-x86_64.tar.gz --strip-components=1 -C qan-api && \
+	tar zxf percona-qan-api.tar.gz --strip-components=1 -C qan-api && \
 	mkdir qan-app && \
-	tar zxf percona-qan-app-1.0.0.tar.gz --strip-components=1 -C qan-app
+	tar zxf percona-qan-app.tar.gz --strip-components=1 -C qan-app
 COPY install-qan.sh /opt
 RUN /opt/install-qan.sh
 
 COPY pt-archiver /usr/bin/
 COPY purge-qan-data /etc/cron.daily
 RUN rm /etc/cron.daily/apt
 
-# ################ #
-# prom-config-api  #
-# ################ #
+# ###### #
+# Consul #
+# ###### #
 
-COPY prom-config-api /opt/prometheus
-RUN mkdir /opt/prometheus/targets
+ADD https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip /opt/
+RUN unzip consul_0.6.4_linux_amd64.zip && \
+	mkdir -p /opt/consul-data
+
+# ##### #
+# Nginx #
+# ##### #
+
+COPY nginx.conf /etc/nginx
 
 # ############ #
 # Landing page # 

add-grafana-datasource.sh

@@ -4,7 +4,7 @@ service grafana-server start
 
 for i in `seq 30`; do
 	if curl -s http://admin:admin@localhost:3000/api/datasources; then
-		curl http://admin:admin@localhost:3000/api/datasources -X POST -H 'Content-Type: application/json' --data-binary '{"name":"Prometheus","type":"prometheus","url":"http://localhost:9090","access":"proxy","isDefault":true}'
+		curl http://admin:admin@localhost:3000/api/datasources -X POST -H 'Content-Type: application/json' --data-binary '{"name":"Prometheus","type":"prometheus","url":"http://localhost:9090/prometheus/","access":"proxy","isDefault":true}'
 		echo "Added Prometheus data source to Grafana"
 		break
 	else

docker-compose.yml

@@ -1,25 +1,26 @@
 version: '2'
 
 services:
-  pmm_data:
+  pmm-data:
+    build: .
     image: percona/pmm-server
-    container_name: pmm_data
+    container_name: pmm-data
     volumes:
       - /var/lib/mysql
       - /opt/prometheus/data
-      - /opt/prometheus/targets
+      - /opt/consul-data
     entrypoint: /bin/true
 
-  pmm:
+  pmm-server:
     build: .
-    container_name: pmm 
+    container_name: pmm-server
     image: percona/pmm-server
-    environment:
-      - ADDRESS=192.168.56.107
     ports:
-      - "9001-9003:9001-9003"
-      - "3000:3000"
-      - "9090:9090"
+      - "9001:9001"
       - "80:80"
     volumes_from:
-      - pmm_data
+      - pmm-data
+    ### EDIT WITH YOUR IP ###
+    environment:
+      - ADDRESS=192.168.56.107
+

grafana.ini

@@ -15,7 +15,12 @@
 # Directory where grafana can store logs
 #
 ;logs = /var/log/grafana
+#
+# Directory where grafana will automatically scan and look for plugins
+#
+;plugins = /var/lib/grafana/plugins
 
+#
 #################################### Server ####################################
 [server]
 # Protocol (http or https)
@@ -25,7 +30,7 @@
 ;http_addr =
 
 # The http port  to use
-;http_port =
+;http_port = 3000
 
 # The public facing domain name used to access grafana from a browser
 ;domain = localhost
@@ -36,6 +41,7 @@
 
 # The full public facing url
 ;root_url = %(protocol)s://%(domain)s:%(http_port)s/
+root_url = %(protocol)s://%(domain)s:%(http_port)s/graph
 
 # Log web requests
 ;router_logging = false
@@ -95,6 +101,13 @@
 # Change this option to false to disable reporting.
 ;reporting_enabled = true
 
+# Set to false to disable all checks to https://grafana.net
+# for new vesions (grafana itself and plugins), check is used
+# in some UI views to notify that grafana or plugin update exists
+# This option does not cause any auto updates, nor send any information
+# only a GET request to http://grafana.net to get latest versions
+;check_for_updates = true
+
 # Google Analytics universal tracking code, only enabled if you specify an id here
 ;google_analytics_ua_id =
 
@@ -120,6 +133,12 @@
 # data source proxy whitelist (ip_or_domain:port seperated by spaces)
 ;data_source_proxy_whitelist =
 
+[snapshots]
+# snapshot sharing options
+;external_enabled = true
+;external_snapshot_url = https://snapshots-origin.raintank.io
+;external_snapshot_name = Publish to snapshot.raintank.io
+
 #################################### Users ####################################
 [users]
 # disable user signup / registration
@@ -205,14 +224,14 @@
 
 #################################### Logging ##########################
 [log]
-# Either "console", "file", default is "console"
+# Either "console", "file", "syslog". Default is console and  file
 # Use comma to separate multiple modes, e.g. "console, file"
 ;mode = console, file
 
 # Buffer length of channel, keep it as it is if you don't know what it is.
 ;buffer_len = 10000
 
-# Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace"
+# Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Info"
 ;level = Info
 
 # For "console" mode only

install-qan.sh

@@ -8,6 +8,3 @@ service mysql start
 
 cd /opt/qan-api
 START="no" SYSINT="no" ./install
-
-cd /opt/qan-app
-START="no" SYSINT="no" LISTEN="0.0.0.0:9002" ./install

landing-page/index.html

@@ -81,14 +81,8 @@ <h2 class="margin-top-0 wow fadeIn">Tell Us What You Think</h2>
     <script src="js/jquery-2.2.2.min.js"></script>
 	<script type="text/javascript">
 		$(document).ready(function(){
-			var hostname = window.location.hostname;
-			if (hostname) {
-				$("#qanURL").attr("href", "http://"+hostname+":9002");
-				$("#grafURL").attr("href", "http://"+hostname+":3000");
-			} else {
-				$("#qanURL").attr("href", "http://localhost:9002");
-				$("#grafURL").attr("href", "http://localhost:3000");
-			}
+			$("#qanURL").attr("href", "/qan/");
+			$("#grafURL").attr("href", "/graph");
 		});
 	</script>
 

nginx.conf

@@ -0,0 +1,54 @@
+user www-data;
+worker_processes 4;
+pid /run/nginx.pid;
+
+events {
+	worker_connections 768;
+}
+
+http {
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+	gzip on;
+	server {
+		listen                  *:80;
+		server_name             _;
+		server_tokens           off;
+		root                    /opt/landing-page; 
+
+		# Grafana
+		rewrite				^/graph$ /graph/;
+		location /graph {
+			proxy_pass		http://localhost:3000;
+			rewrite			^/graph/(.*) /$1 break;
+		}
+
+		# Prometheus
+		location /prometheus {
+			proxy_pass		http://localhost:9090;
+		}
+
+        	# Consul UI
+		location /consul/ {
+        	        proxy_pass              http://localhost:8500/ui/;
+        	}
+
+		# Consul API
+                location /v1/ {
+                        proxy_pass              http://localhost:8500/v1/;
+                }
+
+		# QAN App
+		location /qan {
+			alias			/opt/qan-app;
+		}
+	}
+}
+

prometheus.yml

@@ -5,26 +5,82 @@ global:
 
 scrape_configs:
   - job_name: prometheus
+    metrics_path: /prometheus/metrics
     target_groups:
       - targets: ['localhost:9090']
 
-  - job_name: linux
-    file_sd_configs:
-    - names: ['/opt/prometheus/targets/9100.yml']
+
+  - job_name: override 
+    consul_sd_configs:
+    - server:   'localhost:8500'
+      services: ['linux', 'mongodb']
+
+    relabel_configs:
+    - source_labels: ['__meta_consul_service']
+      regex:         '(.*)'
+      target_label:  'job'
+      replacement:   '$1'
+    - source_labels: ['__meta_consul_node']
+      regex:         '(.*)'
+      target_label:  'alias'
+      replacement:   '$1'
+    - source_labels: ['__meta_consul_tags']
+      regex:         '.*,replset_(\w+),.*'
+      target_label:  'replset'
+      replacement:   '$1'
+    - source_labels: ['__meta_consul_tags']
+      regex:         '.*,cluster_(\w+),.*'
+      target_label:  'cluster'
+      replacement:   '$1'
+    - source_labels: ['__meta_consul_tags']
+      regex:         '.*,nodetype_(\w+),.*'
+      target_label:  'nodetype'
+      replacement:   '$1'
+
 
   - job_name: mysql-hr
-    file_sd_configs:
-    - names: ['/opt/prometheus/targets/9104.yml']
+    consul_sd_configs:
+    - server:   'localhost:8500'
+      services: ['mysql-hr']
+
+    relabel_configs:
+    - target_label:  'job'
+      replacement:   'mysql'
+    - source_labels: ['__meta_consul_node']
+      regex:         '(.*)'
+      target_label:  'alias'
+      replacement:   '$1'
+
 
   - job_name: mysql-mr
     scrape_interval:    5s
     scrape_timeout:     1s
-    file_sd_configs:
-    - names: ['/opt/prometheus/targets/9105.yml']
+    consul_sd_configs:
+    - server:   'localhost:8500'
+      services: ['mysql-mr']
+
+    relabel_configs:
+    - target_label:  'job'
+      replacement:   'mysql'
+    - source_labels: ['__meta_consul_node']
+      regex:         '(.*)'
+      target_label:  'alias'
+      replacement:   '$1'
+
 
   - job_name: mysql-lr
     scrape_interval:   60s
     scrape_timeout:     5s
-    file_sd_configs:
-    - names: ['/opt/prometheus/targets/9106.yml']
+
+    consul_sd_configs:
+    - server:   'localhost:8500'
+      services: ['mysql-lr']
+
+    relabel_configs:
+    - target_label:  'job'
+      replacement:   'mysql'
+    - source_labels: ['__meta_consul_node']
+      regex:         '(.*)'
+      target_label:  'alias'
+      replacement:   '$1'