feat: Implement Tags feature

lib/storage/providers/redis.js

@@ -1,13 +1,11 @@
-var util = require('util');
-var async = require('async');
-var debug = require('debug')('redis-storage');
 var redis = require("redis");
 var shortid = require('shortid');
 var aggregator = require('../../aggregator');
 var connectRedis = require('connect-redis');
 
 var SERVICE_KEY_SUFIX = "service";
 var SERVICES_KEY_SUFIX = "service";
+var TAGS_KEY_SUFIX = "tags";
 var LATENCY_KEY_SUFIX = "latency";
 var CURRENT_OUTAGE_KEY_SUFIX = "outages:current";
 var OUTAGES_KEY_SUFIX = "outages";
@@ -19,9 +17,37 @@ function StorageRedis(options) {
   this.redis.select(this.options.db || 0);
 }
 
+/**
+ * Remove duplicates, change the string of tags to an array,
+ * and updates the main list of tags
+ */
+function handleTags(service, multi) {
+    if (service.tags) {
+        if (!(service.tags instanceof Array)) {
+            service.tags = service.tags.split(",");
+        }
+    } else {
+        service.tags = []
+    }
+
+    service.tags.forEach(function(tag) {
+        multi.sadd(TAGS_KEY_SUFIX, tag.trim());
+    });
+
+    // prevent duplicate tags
+    var finalTags = [];
+    service.tags.forEach(function (tag) {
+        var t = tag.trim();
+        if (!finalTags.indexOf(t) >= 0) finalTags.push(t)
+    });
+    service.tags = finalTags;
+    return service;
+}
+
 exports = module.exports = StorageRedis;
 
 
+
 /**
  * Add service
  * @param service
@@ -33,8 +59,12 @@ StorageRedis.prototype.addService = function (service, callback) {
   service.id = id;
   service.created = +new Date();
   var multi = this.redis.multi();
-  multi.set(SERVICE_KEY_SUFIX + ':' + id, JSON.stringify(service));
   multi.sadd(SERVICES_KEY_SUFIX, id);
+
+  service = handleTags(service, multi);
+
+  multi.set(SERVICE_KEY_SUFIX + ':' + id, JSON.stringify(service));
+
   multi.exec(function (err) {
     callback(err, id);
   });
@@ -50,6 +80,9 @@ StorageRedis.prototype.updateService = function (service, callback) {
   var self = this;
   var multi = this.redis.multi();
   multi.set(SERVICE_KEY_SUFIX + ':' + service.id, JSON.stringify(service));
+
+  handleTags(service, multi);
+
   multi.exec(function (err) {
     if (err) {
       return callback(err);
@@ -118,14 +151,26 @@ StorageRedis.prototype.getServices = function (options, callback) {
       return callback(err);
     }
 
+    function filterByTag(service) {
+        if (!options.tags || !options.tags.length) return true;
+
+        if (!(options.tags instanceof Array)) {
+          return service.tags.indexOf(options.tags) >= 0;
+        }
+        for (var i = 0; i < service.tags.length; i++) {
+          if (options.tags.indexOf(service.tags[i]) >= 0) return true;
+        }
+        return false;
+    }
+
     var multi = self.redis.multi();
     for (var i = 0; i < ids.length; i++) {
       multi.get(SERVICE_KEY_SUFIX + ':' + ids[i]);
     }
     multi.exec(function (err, services) {
       callback(err, services.map(function (service) {
         return JSON.parse(service);
-      }));
+      }).filter(filterByTag));
     });
   });
 };
@@ -252,6 +297,14 @@ StorageRedis.prototype.getLatencySince = function (service, timestamp, aggregate
   });
 };
 
+StorageRedis.prototype.getTags = function(callback) {
+    var self = this;
+    this.redis.smembers(TAGS_KEY_SUFIX, function (err, tags) {
+        return callback(err, tags);
+    });
+
+}
+
 StorageRedis.prototype.resetOutageFailureCount = function (service, cb) {
   this.redis.del(service.id + ':' + FAILURE_COUNT_SUFIX, cb);
 };
@@ -276,6 +329,7 @@ StorageRedis.prototype.getSessionStore = function (session, options) {
   });
 };
 
+
 /**
  * Converts ["124", "1428222697560", "123", "1428222692345"]
  *

scripts/load-json.js

@@ -0,0 +1,120 @@
+var fs = require('fs');
+var async = require('async');
+var colors = require('colors');
+var program = require('commander');
+var pluginLoader = require('../lib/plugin-loader');
+var storageFactory = require('../lib/storage/storage-factory');
+var WatchMenFactory = require('../lib/watchmen');
+var sentinelFactory = require('../lib/sentinel');
+var serviceValidator = require('../lib/service-validator');
+
+var RETURN_CODES = {
+    OK: 0,
+    BAD_STORAGE: 1,
+    GENERIC_ERROR: 2
+};
+
+function exit(code) {
+    process.exit(code);
+}
+
+program
+    .option('-e, --env [env]', 'Storage environment key', process.env.NODE_ENV || 'development')
+    .option('-f, --file [value]', 'File to load', 'data.json')
+    .parse(process.argv);
+
+var storage = storageFactory.getStorageInstance(program.env);
+if (!storage) {
+    console.error('Error creating storage for env: ', program.env);
+    return process.exit(RETURN_CODES.BAD_STORAGE);
+}
+
+// console.log(program.file);
+
+function isIn(element, list, comparator) {
+    for (var i = 0; i < list.length; i++) {
+        if (comparator(element, list[0])) return true;
+    }
+    return false;
+}
+
+function populate(services, storage, callback){
+
+    if (!services || !services.length) {
+        return callback('services not provided');
+    }
+
+    function addService(service, cb) {
+        var errors = serviceValidator.validate(service);
+        if (errors.length === 0){
+            storage.addService(service, cb);
+        } else {
+            cb(errors);
+        }
+    }
+    async.eachSeries(services, addService, function (err) {
+        if (err) {
+            return callback(err);
+        }
+        callback();
+    });
+}
+
+// storage.flush_database(function() { exit(1) });
+
+fs.readFile(__dirname + '/' + program.file, function (err, data) {
+    if (err) throw err;
+
+    var newServices = JSON.parse(data.toString());
+    storage.getServices({}, function(err, services){
+        if (err) throw err;
+
+        var servicesToAdd = [];
+        for (var i = 0; i < newServices.length; i++) {
+            var newService = newServices[i];
+            if (!isIn(newService, services, function(e1, e2) { return e1.name === e2.name })) {
+                console.log('Appending ' + newService.name);
+                servicesToAdd.push(newService);
+                newService.pingServiceOptions = {};
+                var options = {};
+                newService.pingServiceOptions[newService.pingServiceName] = options;
+
+                if (newService.pingServiceName === 'http-contains') {
+                    options['contains'] = {
+                        descr : "response body must contain",
+                        required : true,
+                        value : newService.contains
+                    };
+
+                    options['notContains'] = {
+                        descr : "response body must NOT contain",
+                        required : false,
+                        value : newService.notContains
+                    };
+                    delete newService.notContains;
+                    delete newService.contains;
+                } else {
+                    options['statusCode'] = {
+                        descr : "Expected status code (defaults to 200)",
+                        required : false,
+                        value : newService.statusCode
+                    }
+                    delete newService.statusCode;
+
+                }
+            }
+        }
+
+        if (servicesToAdd.length === 0) {
+            console.log('No new services');
+            exit(RETURN_CODES.OK);
+        } else {
+            populate(servicesToAdd, storage, function(err) {
+                console.log(err);
+                if (err) throw err;
+                exit(RETURN_CODES.OK);
+            })
+        }
+    });
+});
+

test/fixtures/dummy-services.js

@@ -11,6 +11,15 @@ exports = module.exports = (function(){
     }
   }
 
+  function getRandomTags() {
+    var tags = [];
+    var size = faker.random.number(3);
+    for (var i = 0; i < size; i++) {
+      tags.push(faker.random.array_element(['prod', 'dev', 'remote', 'office2']))
+    }
+    return tags;
+  }
+
   function generateDummyService (i){
     return {
       name: getRandomName(false),
@@ -20,7 +29,8 @@ exports = module.exports = (function(){
       port: 443,
       timeout: 10000,
       warningThreshold: 3000,
-      pingServiceName: 'http-head'
+      pingServiceName: 'http-head',
+      tags: getRandomTags()
     };
   }
 

test/test-report.js

@@ -49,7 +49,7 @@ describe('reporting', function () {
 
     reporter = new reportService(storage);
 
-    servicesFixtures = dummyServiceGenerator.generate(1);
+    var servicesFixtures = dummyServiceGenerator.generate(1);
 
     storage.flush_database(function () {
 
@@ -197,11 +197,12 @@ describe('reporting', function () {
           clock.tick(4 * HOUR);
           reporter.getService(services[0].id, function (err, data) {
             assert.ifError(err);
-            assert.equal(data.status.lastWeek.latency.list.length, 4);
-            assert.equal(data.status.lastWeek.latency.list[0].l, 1000);
-            assert.equal(data.status.lastWeek.latency.list[1].l, 400);
-            assert.equal(data.status.lastWeek.latency.list[2].l, 200);
-            assert.equal(data.status.lastWeek.latency.list[3].l, 100);
+            assert.equal(data.status.lastWeek.latency.list.length, 5);
+            assert.equal(data.status.lastWeek.latency.list[0].l, 1100);
+            assert.equal(data.status.lastWeek.latency.list[1].l, 900);
+            assert.equal(data.status.lastWeek.latency.list[2].l, 400);
+            assert.equal(data.status.lastWeek.latency.list[3].l, 200);
+            assert.equal(data.status.lastWeek.latency.list[4].l, 100);
             done();
           });
         });

test/test-storage-redis.js

@@ -32,29 +32,34 @@ describe('redis storage', function(){
   describe('services', function(){
     it('should save and retrieve service object', function(done) {
       var newService = {
-        interval: 1000
+        interval: 1000,
+        tags: 'tag1, tag2'
       };
       redisStorage.addService(newService, function(err, id){
         redisStorage.getService(id, function(err, service) {
           assert.equal(service.id, id);
           assert.equal(service.created, INITIAL_TIME);
-          assert.equal(service.interval, 1000);
+          assert.deepEqual(service.tags, [ 'tag1' , 'tag2' ]);
           done();
         });
       });
     });
 
     it('should update service', function(done) {
       var newService = {
-        interval: 1000
+        interval: 1000,
+        tags: ['tag1', 'tag2', 'tag1']
       };
       redisStorage.addService(newService, function(err, id){
         redisStorage.getService(id, function(err, service) {
           assert.equal(service.id, id);
           assert.equal(service.interval, 1000);
+          assert.deepEqual(service.tags, [ 'tag1' , 'tag2' ]);
           service.interval = 2000;
+          service.tags.push('tag3')
           redisStorage.updateService(service, function(err, service) {
             assert.equal(service.interval, 2000);
+            assert.deepEqual(service.tags, [ 'tag1' , 'tag2', 'tag3' ]);
             done();
           });
         });

webserver/public/js/controllers/service-add.js

@@ -29,6 +29,7 @@
       $scope.service.failuresToBeOutage = 1;
       $scope.service.port = 80;
       $scope.service.pingServiceName = 'http-head';
+      $scope.service.tags = '';
 
       $scope.save = function () {