From ca067fdc5e70d295ff146d655a40706f981bee36 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 25 Apr 2017 11:06:57 -0400 Subject: [PATCH] Use custom VCL for Pass/!Pass conditions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ve now exceeded our max number of characters for a condition in the Fastly API, and we need to make it larger to accommodate regex conditionals that can match on any of the routes in www currently. This fixes the issue by moving the conditions – and the states that are affected by it, like setting the backend or cache ttls – to two custom vcl files that are updated via the Fastly API. One is for the `vcl_rev` config, and one is for the `vcl_fetch` config. --- bin/configure-fastly.js | 83 ++++++++-------------- bin/lib/fastly-extended.js | 89 ++++++++++++++++++++++++ test/unit/test_fastly_extended_method.js | 43 ++++++++++++ 3 files changed, 161 insertions(+), 54 deletions(-) create mode 100644 test/unit/test_fastly_extended_method.js diff --git a/bin/configure-fastly.js b/bin/configure-fastly.js index f64db9a50..4261c6daa 100644 --- a/bin/configure-fastly.js +++ b/bin/configure-fastly.js @@ -8,9 +8,6 @@ var route_json = require('../src/routes.json'); const FASTLY_SERVICE_ID = process.env.FASTLY_SERVICE_ID || ''; const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME || ''; -const PASS_REQUEST_CONDITION_NAME = 'Pass'; -const NOT_PASS_REQUEST_CONDITION_NAME = '!(Pass)'; -const PASS_CACHE_CONDITION_NAME = 'Cache Pass'; const BUCKET_NAME_HEADER_NAME = 'Bucket name'; var fastly = require('./lib/fastly-extended')(process.env.FASTLY_API_KEY, FASTLY_SERVICE_ID); @@ -121,15 +118,35 @@ async.auto({ } }); }, - notPassRequestCondition: ['version', function (cb, results) { - var statement = getAppRouteCondition('../build/*', routes, extraAppRoutes); - var condition = { - name: NOT_PASS_REQUEST_CONDITION_NAME, - statement: statement, - type: 'REQUEST', - priority: 10 - }; - fastly.setCondition(results.version, condition, cb); + recvCustomVCL: ['version', function (cb, results) { + // For all the routes in routes.json, construct a varnish-style regex that matches + // on any of those route conditions. + var notPassStatement = getAppRouteCondition('../build/*', routes, extraAppRoutes); + + // For all the routes in routes.json, construct a varnish-style regex that matches + // only if NONE of those routes are matched. + var passStatement = negateConditionStatement(notPassStatement); + + // For a non-pass condition, point backend at s3 + var backendCondition = fastly.setBackend( + 'F_s3', + S3_BUCKET_NAME, + notPassStatement + ); + // For a pass condition, set forwarding headers + var forwardCondition = fastly.setForwardHeaders(passStatement); + + fastly.setCustomVCL( + results.version, + 'recv-condition', + backendCondition + forwardCondition, + cb + ); + }], + fetchCustomVCL: ['version', function (cb, results) { + var passStatement = negateConditionStatement(getAppRouteCondition('../build/*', routes, extraAppRoutes)); + var ttlCondition = fastly.setResponseTTL(passStatement); + fastly.setCustomVCL(results.version, 'fetch-condition', ttlCondition, cb); }], setBucketNameHeader: ['version', 'notPassRequestCondition', function (cb, results) { var header = { @@ -144,48 +161,6 @@ async.auto({ }; fastly.setFastlyHeader(results.version, header, cb); }], - passRequestCondition: ['version', 'notPassRequestCondition', function (cb, results) { - var condition = { - name: PASS_REQUEST_CONDITION_NAME, - statement: negateConditionStatement(results.notPassRequestCondition.statement), - type: 'REQUEST', - priority: 10 - }; - fastly.setCondition(results.version, condition, cb); - }], - passRequestSettingsCondition: ['version', 'passRequestCondition', function (cb, results) { - fastly.request( - 'PUT', - fastly.getFastlyAPIPrefix(FASTLY_SERVICE_ID, results.version) + '/request_settings/Pass', - {request_condition: results.passRequestCondition.name}, - cb - ); - }], - backendCondition: ['version', 'notPassRequestCondition', function (cb, results) { - fastly.request( - 'PUT', - fastly.getFastlyAPIPrefix(FASTLY_SERVICE_ID, results.version) + '/backend/s3', - {request_condition: results.notPassRequestCondition.name}, - cb - ); - }], - passCacheCondition: ['version', 'passRequestCondition', function (cb, results) { - var condition = { - name: PASS_CACHE_CONDITION_NAME, - type: 'CACHE', - statement: results.passRequestCondition.statement, - priority: results.passRequestCondition.priority - }; - fastly.setCondition(results.version, condition, cb); - }], - passCacheSettingsCondition: ['version', 'passCacheCondition', function (cb, results) { - fastly.request( - 'PUT', - fastly.getFastlyAPIPrefix(FASTLY_SERVICE_ID, results.version) + '/cache_settings/Pass', - {cache_condition: results.passCacheCondition.name}, - cb - ); - }], appRouteRequestConditions: ['version', function (cb, results) { var conditions = {}; async.forEachOf(routes, function (route, id, cb2) { diff --git a/bin/lib/fastly-extended.js b/bin/lib/fastly-extended.js index 14cf0e2a5..f2c3154ce 100644 --- a/bin/lib/fastly-extended.js +++ b/bin/lib/fastly-extended.js @@ -166,5 +166,94 @@ module.exports = function (apiKey, serviceId) { this.request('PUT', url, cb); }; + /** + * Upsert a custom vcl file. Attempts a PUT, and falls back + * to POST if not there already. + * + * @param {number} version current version number for fastly service + * @param {string} name name of the custom vcl file to be upserted + * @param {string} vcl stringified custom vcl to be uploaded + * @param {Function} cb function that takes in two args: err, response + */ + fastly.setCustomVCL = function (version, name, vcl, cb) { + if (!this.serviceId) { + return cb('Failed to set response object. No serviceId configured'); + } + + var url = this.getFastlyAPIPrefix(this.serviceId, version) + '/vcl/' + name; + var postUrl = this.getFastlyAPIPrefix(this.serviceId, version) + '/vcl'; + var content = {content: vcl}; + return this.request('PUT', url, content, function (err, response) { + if (err && err.statusCode === 404) { + content.name = name; + this.request('POST', postUrl, content, function (err, response) { + if (err) { + return cb('Failed while adding custom vcl \"' + name + '\": ' + err); + } + return cb(null, response); + }); + return; + } + if (err) { + return cb('Failed to update custom vcl \"' + name + '\": ' + err); + } + return cb(null, response); + }.bind(this)); + }; + + /** + * Returns custom vcl configuration as a string for setting the backend + * of a request to the given backend/host. + * + * @param {string} backend name of the backend declared in fastly + * @param {string} host name of the s3 bucket to be set as the host + * @param {string} condition condition under which backend should be set + */ + fastly.setBackend = function (backend, host, condition) { + return '' + + 'if (' + condition + ') {\n' + + ' set req.backend = ' + backend + ';\n' + + ' set req.http.host = \"' + host + '\";\n' + + '}\n'; + }; + + /** + * Returns custom vcl configuration as a string for headers that + * should be added for the condition in which a request is forwarded. + * + * @param {string} condition condition under which to set pass headers + */ + fastly.setForwardHeaders = function (condition) { + return '' + + 'if (' + condition + ') {\n' + + ' if (!req.http.Fastly-FF) {\n' + + ' if (req.http.X-Forwarded-For) {\n' + + ' set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For \", \" client.ip;\n' + + ' } else {\n' + + ' set req.http.Fastly-Temp-XFF = client.ip;\n' + + ' }\n' + + ' } else {\n' + + ' set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For;\n' + + ' }\n' + + ' set req.grace = 60s;\n' + + ' return(pass);\n' + + '}\n'; + }; + + /** + * Returns custom vcl configuration as a string that sets the varnish + * Time to Live (TTL) for responses that come from s3. + * + * @param {string} condition condition under which the response should be set + */ + fastly.setResponseTTL = function (condition) { + return '' + + 'if (' + condition + ') {\n' + + ' set beresp.ttl = 0s;\n' + + ' set beresp.grace = 0s;\n' + + ' return(pass);\n' + + '}\n'; + }; + return fastly; }; diff --git a/test/unit/test_fastly_extended_method.js b/test/unit/test_fastly_extended_method.js new file mode 100644 index 000000000..a2bf5144a --- /dev/null +++ b/test/unit/test_fastly_extended_method.js @@ -0,0 +1,43 @@ +var fastly = require('../../bin/lib/fastly-extended'); +var tap = require('tap'); + +tap.test('testSetBackend', function (t) { + var backend = fastly.setBackend('wemust', 'goback', 'marty'); + t.equal(backend, '' + + 'if (marty) {\n' + + ' set req.backend = wemust;\n' + + ' set req.http.host = \"goback\";\n' + + '}\n' + ); +}); + +tap.test('testSetForward', function (t) { + var forward = fastly.setForwardHeaders('alwaysforward'); + t.equal(forward, '' + + 'if (alwaysforward) {\n' + + ' if (!req.http.Fastly-FF) {\n' + + ' if (req.http.X-Forwarded-For) {\n' + + ' set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For ", " client.ip;\n' + + ' } else {\n' + + ' set req.http.Fastly-Temp-XFF = client.ip;\n' + + ' }\n' + + ' } else {\n' + + ' set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For;\n' + + ' }\n' + + ' set req.grace = 60s;\n' + + ' return(pass);\n' + + '}\n' + ); + t.end(); +}); + +tap.test('testSetTTL', function (t) { + var ttl = fastly.setResponseTTL('itsactuallyttyl'); + t.equal(ttl, '' + + 'if (itsactuallyttyl) {\n' + + ' set beresp.ttl = 0s;\n' + + ' set beresp.grace = 0s;\n' + + ' return(pass);\n' + + '}\n' + ); +});