From 5e794493342d4012df9b62997b6ae17bb76bb859 Mon Sep 17 00:00:00 2001 From: rschamp Date: Mon, 5 Aug 2019 15:09:36 -0400 Subject: [PATCH 1/3] Add concurrency limits to async requests --- bin/configure-fastly.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bin/configure-fastly.js b/bin/configure-fastly.js index c6f76411d..b4fa9d552 100644 --- a/bin/configure-fastly.js +++ b/bin/configure-fastly.js @@ -8,6 +8,8 @@ var routeJson = require('../src/routes.json'); const FASTLY_SERVICE_ID = process.env.FASTLY_SERVICE_ID || ''; const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME || ''; +const ASYNC_CONCURRENCY = 1; + var fastly = require('./lib/fastly-extended')(process.env.FASTLY_API_KEY, FASTLY_SERVICE_ID); var extraAppRoutes = [ @@ -92,7 +94,7 @@ async.auto({ }], appRouteRequestConditions: ['version', function (results, cb) { var conditions = {}; - async.forEachOf(routes, function (route, id, cb2) { + async.eachLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { var condition = { name: fastlyConfig.getConditionNameForRoute(route, 'request'), statement: 'req.url ~ "' + route.pattern + '"', @@ -112,7 +114,7 @@ async.auto({ }], appRouteHeaders: ['version', 'appRouteRequestConditions', function (results, cb) { var headers = {}; - async.forEachOf(routes, function (route, id, cb2) { + async.eachLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { if (route.redirect) { async.auto({ responseCondition: function (cb3) { @@ -145,7 +147,7 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb3); }] - }, function (err, redirectResults) { + }, ASYNC_CONCURRENCY, function (err, redirectResults) { if (err) return cb2(err); headers[id] = redirectResults.redirectHeader; cb2(null, redirectResults); @@ -213,7 +215,7 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb2); }] - }, function (err, redirectResults) { + }, ASYNC_CONCURRENCY, function (err, redirectResults) { if (err) return cb(err); cb(null, redirectResults); }); @@ -259,12 +261,12 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb2); }] - }, function (err, redirectResults) { + }, ASYNC_CONCURRENCY, function (err, redirectResults) { if (err) return cb(err); cb(null, redirectResults); }); }] -}, function (err, results) { +}, ASYNC_CONCURRENCY, function (err, results) { if (err) throw new Error(err); if (process.env.FASTLY_ACTIVATE_CHANGES) { fastly.activateVersion(results.version, function (e, resp) { From 802c6bff22b5cceaeaa4e52e53a5033a60b524cc Mon Sep 17 00:00:00 2001 From: rschamp Date: Mon, 5 Aug 2019 15:33:46 -0400 Subject: [PATCH 2/3] Add concurrency limits to async requests of --- bin/configure-fastly.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/configure-fastly.js b/bin/configure-fastly.js index b4fa9d552..716040c20 100644 --- a/bin/configure-fastly.js +++ b/bin/configure-fastly.js @@ -94,7 +94,7 @@ async.auto({ }], appRouteRequestConditions: ['version', function (results, cb) { var conditions = {}; - async.eachLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { + async.eachOfLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { var condition = { name: fastlyConfig.getConditionNameForRoute(route, 'request'), statement: 'req.url ~ "' + route.pattern + '"', @@ -114,7 +114,7 @@ async.auto({ }], appRouteHeaders: ['version', 'appRouteRequestConditions', function (results, cb) { var headers = {}; - async.eachLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { + async.eachOfLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { if (route.redirect) { async.auto({ responseCondition: function (cb3) { From 21dccf60fe7f6a254392b05de8443a5b7d70a5aa Mon Sep 17 00:00:00 2001 From: rschamp Date: Mon, 5 Aug 2019 15:58:17 -0400 Subject: [PATCH 3/3] Rename and make concurrency limit configurable --- bin/configure-fastly.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/configure-fastly.js b/bin/configure-fastly.js index 716040c20..25e747139 100644 --- a/bin/configure-fastly.js +++ b/bin/configure-fastly.js @@ -8,7 +8,7 @@ var routeJson = require('../src/routes.json'); const FASTLY_SERVICE_ID = process.env.FASTLY_SERVICE_ID || ''; const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME || ''; -const ASYNC_CONCURRENCY = 1; +const FASTLY_CONCURRENCY = process.env.FASTLY_CONCURRENCY || 1; var fastly = require('./lib/fastly-extended')(process.env.FASTLY_API_KEY, FASTLY_SERVICE_ID); @@ -94,7 +94,7 @@ async.auto({ }], appRouteRequestConditions: ['version', function (results, cb) { var conditions = {}; - async.eachOfLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { + async.eachOfLimit(routes, FASTLY_CONCURRENCY, function (route, id, cb2) { var condition = { name: fastlyConfig.getConditionNameForRoute(route, 'request'), statement: 'req.url ~ "' + route.pattern + '"', @@ -114,7 +114,7 @@ async.auto({ }], appRouteHeaders: ['version', 'appRouteRequestConditions', function (results, cb) { var headers = {}; - async.eachOfLimit(routes, ASYNC_CONCURRENCY, function (route, id, cb2) { + async.eachOfLimit(routes, FASTLY_CONCURRENCY, function (route, id, cb2) { if (route.redirect) { async.auto({ responseCondition: function (cb3) { @@ -147,7 +147,7 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb3); }] - }, ASYNC_CONCURRENCY, function (err, redirectResults) { + }, FASTLY_CONCURRENCY, function (err, redirectResults) { if (err) return cb2(err); headers[id] = redirectResults.redirectHeader; cb2(null, redirectResults); @@ -215,7 +215,7 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb2); }] - }, ASYNC_CONCURRENCY, function (err, redirectResults) { + }, FASTLY_CONCURRENCY, function (err, redirectResults) { if (err) return cb(err); cb(null, redirectResults); }); @@ -261,12 +261,12 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb2); }] - }, ASYNC_CONCURRENCY, function (err, redirectResults) { + }, FASTLY_CONCURRENCY, function (err, redirectResults) { if (err) return cb(err); cb(null, redirectResults); }); }] -}, ASYNC_CONCURRENCY, function (err, results) { +}, FASTLY_CONCURRENCY, function (err, results) { if (err) throw new Error(err); if (process.env.FASTLY_ACTIVATE_CHANGES) { fastly.activateVersion(results.version, function (e, resp) {