Exponentially backed-off respawns with maximum attempts

README.md

@@ -121,6 +121,9 @@ parameters:
   - `title`: (`String`): see `--title` above.
   - `assumeReady`: (`Boolean`): see Worker readiness below.
   - `keepAlive`: (`Boolean`): see `--keepalive` above.
+  - `backoffRespawns`: null to disable exponential backoff respawns.  -1 to disable maximum respawns, and any positive int to set a limit on attempted respawns
+  - `backoffDelay`: delay to first respawn attempt in milliseconds
+  - `backoffMaxDelay`: the maximum delay between respawn attempts in milliseconds
   - `minExpectedLifetime`: (`Number`|`String`): Number of ms a worker is
     expected to live. Don't auto-respawn if a worker dies earlier. Strings
     like `'10s'` are accepted. Defaults to `'20s'`.

examples/hello-world/up.js

@@ -1,4 +1,4 @@
 
 var server = require('http').Server().listen(3000)
 
-require('../../up')(server, __dirname + '/server')
+require('../../../up')(server, __dirname + '/server')

lib/up.js

@@ -7,6 +7,7 @@ var fork = require('child_process').fork
   , eq = require('eq')
   , os = require('os')
   , ms = require('ms')
+  , backoff = require('backoff')
   , env = process.env.NODE_ENV
   , Distributor = require('distribute')
   , EventEmitter = require('events').EventEmitter
@@ -83,6 +84,9 @@ function UpServer (server, file, opts) {
   this.assumeReady = opts.assumeReady === undefined ? true : !!opts.assumeReady;
   this.keepAlive = opts.keepAlive || false;
   this.minExpectedLifetime = ms(opts.minExpectedLifetime != null ? opts.minExpectedLifetime : minExpectedLifetime);
+  this.backoffRespawns = opts.backoffRespawns || null;
+  this.backoffInitDelay = opts.backoffInitDelay || 100;
+  this.backoffMaxDelay = opts.backoffMaxDelay || 10000;
   if (false !== opts.workerPingInterval) {
     this.workerPingInterval = ms(opts.workerPingInterval || '1m');
   }
@@ -188,9 +192,10 @@ UpServer.prototype.spawnWorkers = function (n) {
  * @api public
  */
 
-UpServer.prototype.spawnWorker = function (fn) {
+UpServer.prototype.spawnWorker = function (fn, dontrespawn) {
   var w = new Worker(this)
     , self = this
+    , previousState = w.readyState;
 
   // keep track that we're spawning
   this.spawning.push(w);
@@ -200,7 +205,14 @@ UpServer.prototype.spawnWorker = function (fn) {
       case 'spawned':
         self.spawning.splice(self.spawning.indexOf(w), 1);
         self.workers.push(w);
+        fn && fn(w.readyState);
         self.emit('spawn', w);
+        // wait until minExpectedLifetime has been reached so that failed starts can be restarted
+        setTimeout(function() { 
+          if (w.readyState == 'terminating' || w.readyState == 'terminated') { 
+            self.emit('unsuccessful', w);
+          } 
+        }, self.minExpectedLifetime-1); 
         break;
 
       case 'terminating':
@@ -212,21 +224,69 @@ UpServer.prototype.spawnWorker = function (fn) {
           self.workers.splice(self.workers.indexOf(w), 1);
           self.lastIndex = -1;
           if (self.keepAlive && (self.workers.length + self.spawning.length < self.numWorkers)) {
-            if (new Date().getTime() - w.birthtime < self.minExpectedLifetime) {
-              debug('worker %s found dead at a too young age. won\'t respawn new worker', w.pid);
+            if (w.uptime() < self.minExpectedLifetime) {
+              debug('worker %s found dead at a too young an age. won\'t respawn new worker', w.pid);
             }
             else {
               debug('worker %s found dead. spawning 1 new worker', w.pid);
-              self.spawnWorker();
+              if (!dontrespawn) self.respawnWorker();
             }
           }
+          fn && fn(w.readyState);
+        } 
+        else if (previousState == 'spawning' && w.proc.exitCode > 0) {
+          if (self.minExpectedLifetime == 0 && self.keepAlive && (self.workers.length + self.spawning.length < self.numWorkers)) { // special case
+            debug('worker %s found dead. spawning 1 new worker', w.pid);
+            if (!dontrespawn) self.respawnWorker();
+          }
+          else {
+            self.emit('unsuccessful', w);
+          }
         }
-        self.emit('terminate', w)
+        self.emit('terminate', w);
         break;
     }
+    previousState = w.readyState;
   });
 };
 
+/**
+ * Spawns a worker that binds to an available port.
+ *
+ * @api private
+ */
+
+UpServer.prototype.respawnWorker = function () {
+  var self = this;
+
+  self.emit('respawn');
+
+  if (self.backoffRespawns !== null) {
+    var respawnBackoff = backoff.exponential({
+          initialDelay: self.backoffInitDelay
+        , maxDelay: self.backoffMaxDelay
+      })
+    respawnBackoff.on('ready', function(number, delay) {
+        self.spawnWorker(function(readyState) {
+          if (readyState == 'terminated') {
+            self.emit('respawn');
+            respawnBackoff.backoff();
+          }
+        }, true);
+      })
+    respawnBackoff.on('fail', function() {
+      self.emit('respawnerror');
+    });
+    if (self.backoffRespawns > 0) {
+      respawnBackoff.failAfter(self.backoffRespawns-1);
+    }
+    respawnBackoff.backoff();
+  }
+  else {
+    self.spawnWorker(null, true);
+  }
+};
+
 /**
  * Gets the next port in the round.
  *
@@ -373,6 +433,16 @@ Worker.prototype.shutdown = function () {
   }
 };
 
+/**
+ * Uptime of current worker
+ *
+ * @api public
+ */
+
+Worker.prototype.uptime = function() {
+  return new Date().getTime() - this.birthtime;
+};
+
 /**
  * Send ready signal from within a worker
  *

package.json

@@ -12,6 +12,7 @@
       , "ms": "0.1.0"
       , "debug": "0.1.0"
       , "commander": "0.6.1"
+      , "backoff": "2.0.0"
       , "distribute": "0.1.4"
     }
   , "devDependencies": {

test/child-backoff.js

@@ -0,0 +1,4 @@
+
+var httpServer = require('http').Server()
+  , up = require('../lib/up')(httpServer, __dirname + '/child-server-backoff'
+      , { workerPingInterval: '15ms', numWorkers: 1, keepAlive:true, minExpectedLifetime:0, backoffRespawns:-1, backoffInitDelay:10, backoffMaxDelay:1000 })

test/child-server-backoff.js

@@ -0,0 +1,6 @@
+
+var client = require('net').connect(7003, function () {
+  client.write(String(process.pid));
+});
+
+module.exports = require('http').Server()

test/server-asyncfail.js

@@ -0,0 +1,3 @@
+var server = require('./server');
+module.exports = server;
+setTimeout(function() { process.exit(1); }, 100);

test/server-fail.js

@@ -0,0 +1,2 @@
+
+process.exit(1);

test/server.js

@@ -9,7 +9,7 @@ var express = require('express')
  * Initialize server
  */
 
-var app = express.createServer();
+var app = express();
 
 /**
  * Default route.