Use custom VCL for Pass/!Pass conditions

We’ve now exceeded our max number of characters for a condition in the Fastly API, and we need to make it larger to accommodate regex conditionals that can match on any of the routes in www currently.

This fixes the issue by moving the conditions – and the states that are affected by it, like setting the backend or cache ttls – to two custom vcl files that are updated via the Fastly API. One is for the `vcl_rev` config, and one is for the `vcl_fetch` config.

bin/configure-fastly.js

@@ -8,9 +8,6 @@ var route_json = require('../src/routes.json');
 const FASTLY_SERVICE_ID = process.env.FASTLY_SERVICE_ID || '';
 const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME || '';
 
-const PASS_REQUEST_CONDITION_NAME = 'Pass';
-const NOT_PASS_REQUEST_CONDITION_NAME = '!(Pass)';
-const PASS_CACHE_CONDITION_NAME = 'Cache Pass';
 const BUCKET_NAME_HEADER_NAME = 'Bucket name';
 
 var fastly = require('./lib/fastly-extended')(process.env.FASTLY_API_KEY, FASTLY_SERVICE_ID);
@@ -121,15 +118,35 @@ async.auto({
             }
         });
     },
-    notPassRequestCondition: ['version', function (cb, results) {
+    recvCustomVCL: ['version', function (cb, results) {
-        var statement = getAppRouteCondition('../build/*', routes, extraAppRoutes);
+        // For all the routes in routes.json, construct a varnish-style regex that matches
-        var condition = {
+        // on any of those route conditions.
-            name: NOT_PASS_REQUEST_CONDITION_NAME,
+        var notPassStatement = getAppRouteCondition('../build/*', routes, extraAppRoutes);
-            statement: statement,
+        
-            type: 'REQUEST',
+        // For all the routes in routes.json, construct a varnish-style regex that matches
-            priority: 10
+        // only if NONE of those routes are matched.
-        };
+        var passStatement = negateConditionStatement(notPassStatement);
-        fastly.setCondition(results.version, condition, cb);
+        
+        // For a non-pass condition, point backend at s3
+        var backendCondition = fastly.setBackend(
+            'F_s3',
+            S3_BUCKET_NAME,
+            notPassStatement
+        );
+        // For a pass condition, set forwarding headers
+        var forwardCondition = fastly.setForwardHeaders(passStatement);
+
+        fastly.setCustomVCL(
+            results.version,
+            'recv-condition',
+            backendCondition + forwardCondition,
+            cb
+        );
+    }],
+    fetchCustomVCL: ['version', function (cb, results) {
+        var passStatement = negateConditionStatement(getAppRouteCondition('../build/*', routes, extraAppRoutes));
+        var ttlCondition = fastly.setResponseTTL(passStatement);
+        fastly.setCustomVCL(results.version, 'fetch-condition', ttlCondition, cb);
     }],
     setBucketNameHeader: ['version', 'notPassRequestCondition', function (cb, results) {
         var header = {
@@ -144,48 +161,6 @@ async.auto({
         };
         fastly.setFastlyHeader(results.version, header, cb);
     }],
-    passRequestCondition: ['version', 'notPassRequestCondition', function (cb, results) {
-        var condition = {
-            name: PASS_REQUEST_CONDITION_NAME,
-            statement: negateConditionStatement(results.notPassRequestCondition.statement),
-            type: 'REQUEST',
-            priority: 10
-        };
-        fastly.setCondition(results.version, condition, cb);
-    }],
-    passRequestSettingsCondition: ['version', 'passRequestCondition', function (cb, results) {
-        fastly.request(
-            'PUT',
-            fastly.getFastlyAPIPrefix(FASTLY_SERVICE_ID, results.version) + '/request_settings/Pass',
-            {request_condition: results.passRequestCondition.name},
-            cb
-        );
-    }],
-    backendCondition: ['version', 'notPassRequestCondition', function (cb, results) {
-        fastly.request(
-            'PUT',
-            fastly.getFastlyAPIPrefix(FASTLY_SERVICE_ID, results.version) + '/backend/s3',
-            {request_condition: results.notPassRequestCondition.name},
-            cb
-        );
-    }],
-    passCacheCondition: ['version', 'passRequestCondition', function (cb, results) {
-        var condition = {
-            name: PASS_CACHE_CONDITION_NAME,
-            type: 'CACHE',
-            statement: results.passRequestCondition.statement,
-            priority: results.passRequestCondition.priority
-        };
-        fastly.setCondition(results.version, condition, cb);
-    }],
-    passCacheSettingsCondition: ['version', 'passCacheCondition', function (cb, results) {
-        fastly.request(
-            'PUT',
-            fastly.getFastlyAPIPrefix(FASTLY_SERVICE_ID, results.version) + '/cache_settings/Pass',
-            {cache_condition: results.passCacheCondition.name},
-            cb
-        );
-    }],
     appRouteRequestConditions: ['version', function (cb, results) {
         var conditions = {};
         async.forEachOf(routes, function (route, id, cb2) {

bin/lib/fastly-extended.js

@@ -166,5 +166,94 @@ module.exports = function (apiKey, serviceId) {
         this.request('PUT', url, cb);
     };
 
+    /**
+     * Upsert a custom vcl file. Attempts a PUT, and falls back
+     * to POST if not there already.
+     *
+     * @param {number}   version current version number for fastly service
+     * @param {string}   name    name of the custom vcl file to be upserted
+     * @param {string}   vcl     stringified custom vcl to be uploaded
+     * @param {Function} cb      function that takes in two args: err, response
+     */
+    fastly.setCustomVCL = function (version, name, vcl, cb) {
+        if (!this.serviceId) {
+            return cb('Failed to set response object. No serviceId configured');
+        }
+
+        var url = this.getFastlyAPIPrefix(this.serviceId, version) + '/vcl/' + name;
+        var postUrl = this.getFastlyAPIPrefix(this.serviceId, version) + '/vcl';
+        var content = {content: vcl};
+        return this.request('PUT', url, content, function (err, response) {
+            if (err && err.statusCode === 404) {
+                content.name = name;
+                this.request('POST', postUrl, content, function (err, response) {
+                    if (err) {
+                        return cb('Failed while adding custom vcl \"' + name + '\": ' + err);
+                    }
+                    return cb(null, response);
+                });
+                return;
+            }
+            if (err) {
+                return cb('Failed to update custom vcl \"' + name + '\": ' + err);
+            }
+            return cb(null, response);
+        }.bind(this));
+    };
+
+    /**
+     * Returns custom vcl configuration as a string for setting the backend
+     * of a request to the given backend/host.
+     *
+     * @param {string} backend   name of the backend declared in fastly
+     * @param {string} host      name of the s3 bucket to be set as the host
+     * @param {string} condition condition under which backend should be set
+     */
+    fastly.setBackend = function (backend, host, condition) {
+        return '' +
+            'if (' + condition + ') {\n' +
+            '    set req.backend = ' + backend + ';\n' +
+            '    set req.http.host = \"' + host + '\";\n' +
+            '}\n';
+    };
+
+    /**
+     * Returns custom vcl configuration as a string for headers that
+     * should be added for the condition in which a request is forwarded.
+     *
+     * @param {string} condition condition under which to set pass headers
+     */
+    fastly.setForwardHeaders = function (condition) {
+        return '' +
+            'if (' + condition + ') {\n' +
+            '    if (!req.http.Fastly-FF) {\n' +
+            '        if (req.http.X-Forwarded-For) {\n' +
+            '            set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For \", \" client.ip;\n' +
+            '        } else {\n' +
+            '            set req.http.Fastly-Temp-XFF = client.ip;\n' +
+            '        }\n' +
+            '    } else {\n' +
+            '        set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For;\n' +
+            '    }\n' +
+            '    set req.grace = 60s;\n' +
+            '    return(pass);\n' +
+            '}\n';
+    };
+
+    /**
+     * Returns custom vcl configuration as a string that sets the varnish
+     * Time to Live (TTL) for responses that come from s3.
+     *
+     * @param {string} condition condition under which the response should be set
+     */
+    fastly.setResponseTTL = function (condition) {
+        return '' +
+            'if (' + condition + ') {\n' +
+            '    set beresp.ttl = 0s;\n' +
+            '    set beresp.grace = 0s;\n' +
+            '    return(pass);\n' +
+            '}\n';
+    };
+
     return fastly;
 };

test/unit/test_fastly_extended_method.js

@@ -0,0 +1,43 @@
+var fastly = require('../../bin/lib/fastly-extended');
+var tap = require('tap');
+
+tap.test('testSetBackend', function (t) {
+    var backend = fastly.setBackend('wemust', 'goback', 'marty');
+    t.equal(backend, '' +
+        'if (marty) {\n' +
+        '    set req.backend = wemust;\n' +
+        '    set req.http.host = \"goback\";\n' +
+        '}\n'
+    );
+});
+
+tap.test('testSetForward', function (t) {
+    var forward = fastly.setForwardHeaders('alwaysforward');
+    t.equal(forward, '' +
+        'if (alwaysforward) {\n' +
+        '    if (!req.http.Fastly-FF) {\n' +
+        '        if (req.http.X-Forwarded-For) {\n' +
+        '            set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For ", " client.ip;\n' +
+        '        } else {\n' +
+        '            set req.http.Fastly-Temp-XFF = client.ip;\n' +
+        '        }\n' +
+        '    } else {\n' +
+        '        set req.http.Fastly-Temp-XFF = req.http.X-Forwarded-For;\n' +
+        '    }\n' +
+        '    set req.grace = 60s;\n' +
+        '    return(pass);\n' +
+        '}\n'
+    );
+    t.end();
+});
+
+tap.test('testSetTTL', function (t) {
+    var ttl = fastly.setResponseTTL('itsactuallyttyl');
+    t.equal(ttl, '' +
+        'if (itsactuallyttyl) {\n' +
+        '    set beresp.ttl = 0s;\n' +
+        '    set beresp.grace = 0s;\n' +
+        '    return(pass);\n' +
+        '}\n'
+    );
+});