Initial commit

.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+.DS_Store

README.md

@@ -0,0 +1,121 @@
+# connect-rate-limiter
+
+Rate-limit your `Node.js` API, backed by Redis.
+
+- easy to configure
+- can set limits for different routes
+- tested under heavy load for race conditions
+
+[![NPM](http://img.shields.io/npm/v/redis-rate-limiter.svg?style=flat)](https://npmjs.org/package/redis-rate-limiter)
+[![License](http://img.shields.io/npm/l/redis-rate-limiter.svg?style=flat)](https://github.com/TabDigital/redis-rate-limiter)
+
+[![Build Status](http://img.shields.io/travis/TabDigital/redis-rate-limiter.svg?style=flat)](http://travis-ci.org/TabDigital/redis-rate-limiter)
+[![Dependencies](http://img.shields.io/david/TabDigital/redis-rate-limiter.svg?style=flat)](https://david-dm.org/TabDigital/redis-rate-limiter)
+[![Dev dependencies](http://img.shields.io/david/dev/TabDigital/redis-rate-limiter.svg?style=flat)](https://david-dm.org/TabDigital/redis-rate-limiter)
+
+The simplest example is
+
+```coffee
+RateLimiter = require 'connect-rate-limiter'
+
+limiter = new RateLimiter(redis: 'redis://localhost:6379')
+server.use limiter.middleware(key: 'ip', rate: '10 req/second')
+```
+
+That's it!
+No one will be able to make more than 10 requests per second from a given IP.
+
+## Events
+
+You can listen to the `accepted` event to apply extra logic, for example adding custom headers.
+
+```coffee
+limiter.on 'accepted', (rate, req, res) ->
+  res.headers('X-RateLimit-Window',  rate.window)   # 60   = 1 minute
+  res.headers('X-RateLimit-Total',   rate.total)    # 100  = 100 req/minute
+  res.headers('X-RateLimit-Current', rate.current)  # 35   = 35 out of 100
+```
+
+By default, rate-limited requests get terminated with a status code of `429`.
+You can listen to the `rejected` event to override this behaviour.
+If you attach an event handler, you **must** terminate the request yourself.
+
+```coffee
+limiter.on 'rejected', (rate, req, res, next) ->
+  res.send 429, 'Too many requests'
+  # or for example
+  next new restify.RequestThrottledError('Too many requests')
+```
+
+Finally, if Redis is not available the middleware won't apply any rate-limiting.
+You can catch that event for logging purposes.
+
+```coffee
+limiter.on 'unavailable', (err) ->
+  console.log 'Failed to rate limit', err
+```
+
+# Rate-limiting key
+
+Rate-limiting is applied per user - which are identified with a unique key.
+There are several helpers built-in:
+
+```coffee
+# identify users by IP
+key: 'ip'
+
+# identify users by their IP network (255.255.255.0 mask)
+key: 'ip/32'
+
+# identify users by the X-Forwarded-For header
+# careful: this is just an HTTP header and can easily be spoofed
+key: 'x-forwarded-for'
+```
+
+You can also specify a custom function to extract the key from the request.
+
+```coffee
+# use your own custom function
+key: (req) -> req.body.account.number
+```
+
+# Request rate
+
+The rate is made of two components.
+
+```coffee
+limit: 100   # 100 requests
+window: 60   # per minute
+```
+
+You can also use a shorthand notation using the `rate` property.
+
+```coffee
+rate: '10 req/second'
+rate: '200 req/minute'
+rate: '5000 req/hour'
+```
+
+# Multiple limiters
+
+You can combine several rate-limiters, either on the entire server or at the route level.
+
+```coffee
+# rate limit the whole server to 10/sec from any IP
+server.use limiter.middleware(key: 'ip', rate: '10 req/second')
+
+# but you also can't create more than 1 user/min from a given IP
+server.post '/api/users',
+  limiter.middleware(key: 'ip', rate: '5 req/minute'),
+  controller.create
+```
+
+You can also apply several limiters with different criteria.
+They will be executed in series, as a logical `AND`.
+
+```coffee
+# no more than 100 requests per IP
+# and no more than 10 requests per account
+server.use limiter.middleware(key: uniqueIp, rate: '100 req/second')
+server.use limiter.middleware(key: accountNumber, rate: '50 req/minute')
+```

lib/index.js

@@ -0,0 +1,52 @@
+var redis        = require('redis');
+var util         = require('util');
+var url          = require('url');
+var EventEmitter = require('events').EventEmitter;
+var options      = require('./options');
+
+function RateLimiter(config) {
+  var cnx = url.parse(config.redis);
+  this.client = redis.createClient(cnx.port, cnx.hostname, {enable_offline_queue: false});
+  this.client.on('error', this.emit.bind(this, 'unavailable'));
+}
+
+util.inherits(RateLimiter, EventEmitter);
+module.exports = RateLimiter;
+
+RateLimiter.prototype.middleware = function(opts) {
+  var self = this;
+  opts = options.canonical(opts);
+  return function(req, res, next) {
+    var key = opts.key(req);
+    var tempKey = 'ratelimittemp:' + key;
+    var realKey = 'ratelimit:' + key;
+    self.client
+        .multi()
+        .setex(tempKey, opts.window, 0)
+        .renamenx(tempKey, realKey)
+        .incr(realKey)
+        .exec(function(err, results) {
+          if(err) {
+            self.emit('unavailable', err);
+            next();
+          } else {
+            var rate = {
+              current: results[2],
+              limit: opts.limit,
+              window: opts.window
+            };
+            if (rate.current <= rate.limit) {
+              self.emit('accepted', rate, req, res, next);
+              next();
+            } else {
+              if (self.listeners('rejected').length === 0) {
+                res.writeHead(429);
+                res.end();
+              } else {
+                self.emit('rejected', rate, req, res, next);
+              }
+            }
+          }
+        });
+  };
+};

lib/options.js

@@ -0,0 +1,42 @@
+var assert  = require('assert');
+var moment  = require('moment');
+var ip      = require('ip');
+
+exports.canonical = function(opts) {
+
+  var canon = {};
+
+  // Key function
+  if (typeof opts.key === 'function') canon.key = opts.key;
+  if (typeof opts.key === 'string')   canon.key = keyShorthands[opts.key];
+
+  // Rate shorthand
+  if (opts.rate) {
+    assert.equal(typeof opts.rate, 'string', 'Invalid rate: ' + opts.rate);
+    var match = opts.rate.match(/^(\d+) req\/([a-z]+)$/);
+    assert.ok(match, 'Invalid rate: ' + opts.rate);
+    canon.limit = parseInt(match[1], 10);
+    canon.window = moment.duration(1, match[2]) / 1000;
+  }
+
+  // Limit + window
+  if (opts.limit)  canon.limit  = opts.limit;
+  if (opts.window) canon.window = opts.window;
+
+  // Check option types
+  assert.equal(typeof canon.key, 'function', 'Invalid key: ' + opts.key);
+  assert.equal(typeof canon.limit, 'number', 'Invalid limit: ' + canon.limit);
+  assert.equal(typeof canon.window, 'number', 'Invalid window: ' + canon.window);
+
+  return canon;
+};
+
+
+var keyShorthands = {
+  'ip': function(req) {
+    return req.connection.remoteAddress;
+  },
+  'ip/32': function (req) {
+    return ip.mask(req.connection.remoteAddress, '255.255.255.0') + '/32';
+  }
+};

package.json

@@ -0,0 +1,24 @@
+{
+  "name": "connect-rate-limiter",
+  "version": "0.0.0",
+  "description": "Rate-limit middleware, backed by Redis",
+  "author": "Tabcorp Digital Team",
+  "license": "MIT",
+  "main": "lib/index.js",
+  "scripts": {
+    "test": "./node_modules/.bin/mocha"
+  },
+  "dependencies": {
+    "ip": "~0.3.1",
+    "moment": "~2.8.1",
+    "redis": "~0.12.1"
+  },
+  "devDependencies": {
+    "async": "~0.9.0",
+    "connect": "~2.25.7",
+    "lodash": "~2.4.1",
+    "mocha": "~1.21.4",
+    "should": "~4.0.4",
+    "supertest": "~0.13.0"
+  }
+}

test/middleware.spec.js

@@ -0,0 +1,145 @@
+var _            = require('lodash');
+var async        = require('async');
+var should       = require('should');
+var connect      = require('connect');
+var supertest    = require('supertest');
+var RateLimiter  = require('../lib/index');
+
+describe('Rate-limit middleware', function() {
+
+  this.slow(5000);
+  this.timeout(5000);
+
+  var limiter = null;
+
+  before(function(done) {
+    limiter = new RateLimiter({redis: 'redis://localhost:6379'});
+    limiter.client.on('connect', done);
+  });
+
+  beforeEach(function(done) {
+    limiter.client.del('ratelimit:127.0.0.1', done);
+  });
+
+  describe('IP throttling', function() {
+
+    it('works under the limit', function(done) {
+      var server = connect();
+      server.use(limiter.middleware({key: 'ip', rate: '10 req/second'}));
+      server.use(fastResponse);
+      var reqs = requests(server, '/test', 9);
+      async.parallel(reqs, function(err, data) {
+        withStatus(data, 200).should.have.length(9);
+        done();
+      });
+    });
+
+    it('fails over the limit', function(done) {
+      var server = connect();
+      server.use(limiter.middleware({key: 'ip', rate: '10 req/second'}));
+      server.use(fastResponse);
+      var reqs = requests(server, '/test', 12);
+      async.parallel(reqs, function(err, data) {
+        withStatus(data, 200).should.have.length(10);
+        withStatus(data, 429).should.have.length(2);
+        done();
+      });
+    });
+
+    it('can go under / over / under', function(done) {
+      var server = connect();
+      server.use(limiter.middleware({key: 'ip', rate: '10 req/second'}));
+      server.use(fastResponse);
+      async.series([
+        function(next) { async.parallel(requests(server, '/test', 9), next); },
+        function(next) { setTimeout(next, 1100); },
+        function(next) { async.parallel(requests(server, '/test', 12), next); },
+        function(next) { setTimeout(next, 1100); },
+        function(next) { async.parallel(requests(server, '/test', 9), next); }
+      ], function(err, data) {
+        withStatus(data[0], 200).should.have.length(9);
+        withStatus(data[2], 200).should.have.length(10);
+        withStatus(data[2], 429).should.have.length(2);
+        withStatus(data[4], 200).should.have.length(9);
+        done();
+      });
+    });
+
+  });
+
+  describe('Custom key throttling', function() {
+
+  });
+
+});
+
+
+
+  // describe 'Account throttling', ->
+  //
+  //   it 'concurrent requests (different accounts)', (done) ->
+  //     server.use authToken
+  //     server.use restify.throttle(username: true, burst: 2, rate: 0)
+  //     server.get '/test', slowResponse
+  //     reqs = [
+  //       (next) -> request(server).get('/test?username=bob').end(next)
+  //       (next) -> request(server).get('/test?username=jane').end(next)
+  //       (next) -> request(server).get('/test?username=john').end(next)
+  //     ]
+  //     async.parallel reqs, (err, data) ->
+  //       withStatus(data, 200).should.have.length 3
+  //       done()
+  //
+  //   it 'concurrent requests (under the limit)', (done) ->
+  //     server.use authToken
+  //     server.use restify.throttle(username: true, burst: 3, rate: 0)
+  //     server.get '/test', slowResponse
+  //     reqs = [
+  //       (next) -> request(server).get('/test').end(next)
+  //       (next) -> request(server).get('/test').end(next)
+  //     ]
+  //     async.parallel reqs, (err, data) ->
+  //       withStatus(data, 200).should.have.length 2
+  //       done()
+  //
+  //   it 'concurrent requests (over the limit)', (done) ->
+  //     server.use authToken
+  //     server.use restify.throttle(username: true, burst: 2, rate: 0)
+  //     server.get '/test', slowResponse
+  //     reqs = [
+  //       (next) -> request(server).get('/test?username=bob').end(next)
+  //       (next) -> request(server).get('/test?username=bob').end(next)
+  //       (next) -> request(server).get('/test?username=bob').end(next)
+  //     ]
+  //     async.parallel reqs, (err, data) ->
+  //       withStatus(data, 200).should.have.length 2
+  //       withStatus(data, 429).should.have.length 1
+  //       done()
+
+function request(server, url) {
+  return function(next) {
+    supertest(server).get('/test').end(next);
+  };
+}
+
+function requests(server, url, count) {
+  return _.times(count, function() {
+    return request(server, url);
+  });
+}
+
+function fastResponse(req, res, next) {
+  res.writeHead(200);
+  res.end('ok');
+}
+
+function withStatus(data, code) {
+  var pretty = data.map(function(d) {
+    return {
+      statusCode: d.res.statusCode,
+      body: d.res.body
+    }
+  });
+  // console.log('pretty', pretty)
+  return _.filter(pretty, {statusCode: code});
+}

test/options.spec.js

@@ -0,0 +1,116 @@
+require('should');
+var options = require('../lib/options');
+
+describe('Options', function() {
+
+  describe('key', function() {
+
+    it('can specify a function', function() {
+      var opts = options.canonical({
+        key: function(req) { return req.id; },
+        limit: 10,
+        window: 60
+      });
+      opts.key({
+        id: 5
+      }).should.eql(5);
+    });
+
+    it('can be the full client IP', function() {
+      var opts = options.canonical({
+        key: 'ip',
+        limit: 10,
+        window: 60
+      });
+      opts.key({
+        connection: { remoteAddress: '1.2.3.4' }
+      }).should.eql('1.2.3.4');
+    });
+
+    it('can be the client IP/32 mask', function() {
+      var opts = options.canonical({
+        key: 'ip/32',
+        limit: 10,
+        window: 60
+      });
+      opts.key({
+        connection: { remoteAddress: '1.2.3.4' }
+      }).should.eql('1.2.3.0/32');
+    });
+
+    it('fails for invalid keys', function() {
+      (function() {
+        var opts = options.canonical({
+          key: 'something',
+          limit: 10,
+          window: 60
+        });
+      }).should.throw('Invalid key: something');
+    });
+
+  });
+
+  describe('limit and window', function() {
+
+    it('should accept numeric values in seconds', function() {
+      var opts = options.canonical({
+        key: 'ip',
+        limit: 10,   // 10 requests
+        window: 60   // per 60 seconds
+      });
+      opts.limit.should.eql(10);
+      opts.window.should.eql(60);
+    });
+
+  });
+
+  describe('rate shorthand notation', function() {
+
+    it('X req/second', function() {
+      var opts = options.canonical({
+        key: 'ip',
+        rate: '10 req/second'
+      });
+      opts.limit.should.eql(10);
+      opts.window.should.eql(1);
+    });
+
+    it('X req/minute', function() {
+      var opts = options.canonical({
+        key: 'ip',
+        rate: '20 req/minute'
+      });
+      opts.limit.should.eql(20);
+      opts.window.should.eql(60);
+    });
+
+    it('X req/hour', function() {
+      var opts = options.canonical({
+        key: 'ip',
+        rate: '1000 req/hour'
+      });
+      opts.limit.should.eql(1000);
+      opts.window.should.eql(3600);
+    });
+
+    it('X req/day', function() {
+      var opts = options.canonical({
+        key: 'ip',
+        rate: '5000 req/day'
+      });
+      opts.limit.should.eql(5000);
+      opts.window.should.eql(86400);
+    });
+
+    it('has to be a valid rate', function() {
+      (function() {
+        var opts = options.canonical({
+          key: 'ip',
+          rate: '50 things'
+        });
+      }).should.throw('Invalid rate: 50 things');
+    });
+
+  });
+
+});