From dd1c2c1627dd8dd503fb8a864224406d91f2934a Mon Sep 17 00:00:00 2001 From: Ray Schamp Date: Fri, 29 Apr 2016 14:33:04 -0400 Subject: [PATCH 1/2] Use s3cmd to sync with S3 --- .gitignore | 1 + .travis.yml | 5 +++- Makefile | 3 ++- README.md | 20 +++++++++++++++- bin/deploy-to-s3.js | 57 --------------------------------------------- package.json | 1 - requirements.txt | 1 + 7 files changed, 27 insertions(+), 61 deletions(-) delete mode 100644 bin/deploy-to-s3.js create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 02d5a403c..87f017249 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ npm-* !.elasticbeanstalk/*.cfg.yml !.elasticbeanstalk/*.global.yml deploy.zip +ENV diff --git a/.travis.yml b/.travis.yml index f651219c1..d883a6158 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: node_js node_js: - '4.2' -sudo: false +sudo: required cache: directories: - node_modules @@ -37,6 +37,9 @@ addons: - ubuntu-toolchain-r-test packages: - g++-4.8 +install: + - sudo -H pip install -r requirements.txt + - npm install deploy: - provider: script skip_cleanup: $SKIP_CLEANUP diff --git a/Makefile b/Makefile index 67ea298cc..8cfececd8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ ESLINT=./node_modules/.bin/eslint NODE=node SASSLINT=./node_modules/.bin/sass-lint -v +S3CMD=s3cmd TAP=./node_modules/.bin/tap WATCH=./node_modules/.bin/watch WEBPACK=./node_modules/.bin/webpack @@ -33,7 +34,7 @@ webpack: $(WEBPACK) --bail sync-s3: - $(NODE) ./bin/deploy-to-s3.js + $(S3CMD) sync -P --delete-removed --exclude '.DS_Store' ./build/ s3://$(S3_BUCKET_NAME)/ sync-fastly: $(NODE) ./bin/configure-fastly.js diff --git a/README.md b/README.md index 8b6f28dfd..9d1fd9cfa 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Use `^C` to stop the node process `npm start` starts. | `API_HOST` | `https://api.scratch.mit.edu` | Hostname for API requests | | `NODE_ENV` | `null` | If not `production`, app acts like development | | `PORT` | `8333` | Port for devserver (http://localhost:XXXX) | -| `FALLBACK` | `''` | Pass-through location for old site | +| `FALLBACK` | `''` | Pass-through location for old site | **NOTE:** Because by default `API_HOST=https://api.scratch.mit.edu`, please be aware that, by default, you will be seeing and interacting with real data on the Scratch website. @@ -61,6 +61,24 @@ Use `^C` to stop the node process `npm start` starts. npm test ``` +### To Deploy +```bash +npm install +virtualenv ENV +. ENV/bin/activate +pip install -r requirements.txt +make deploy +``` + +| Variable | Default | Description | +| ----------------------- | ------- | ----------------------------------------------- | +| `FASTLY_SERVICE_ID` | `''` | Fastly service ID for `bin/configure-fastly.js` | +| `FASTLY_API_KEY` | `''` | Fastly API key for `bin/configure-fastly.js` | +| `AWS_ACCESS_KEY_ID` | `''` | AWS access key id for S3 | +| `AWS_SECRET_ACCESS_KEY` | `''` | AWS secret access key for S3 | +| `S3_BUCKET_NAME` | `''` | S3 bucket name to deploy into | + + ### Current issues with the development We're currently in the process of transitioning into this web client from Scratch's existing structure. As we transition, there are going to be some issues along the way that relate to how this client needs to interact with the existing infrastructure to work properly in production. diff --git a/bin/deploy-to-s3.js b/bin/deploy-to-s3.js deleted file mode 100644 index 2d701009b..000000000 --- a/bin/deploy-to-s3.js +++ /dev/null @@ -1,57 +0,0 @@ -var path = require('path'); -var s3 = require('s3'); -var util = require('util'); - -const IGNORE_FILES = [ - '.DS_Store' -]; - -var s3Client = s3.createClient({ - s3Options: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '' - } -}); - -var localDir = process.env.BUILD_DIRECTORY; -if (!localDir) { - localDir = path.resolve(__dirname, '../build'); -} else { - if (!path.isAbsolute(localDir)) { - localDir = path.join(process.cwd(), localDir); - } -} - -var skipped = 0; -var uploaded = 0; - -var sync = s3Client.uploadDir({ - deleteRemoved: true, - localDir: localDir, - getS3Params: function (localfile, stat, callback) { - if (IGNORE_FILES.indexOf(path.basename(localfile)) != -1) { - skipped++; - return callback(null, null); - } - callback(null, {}); - }, - s3Params: { - Prefix: '', - Bucket: process.env.S3_BUCKET_NAME || '', - ACL: 'public-read' - } -}); - -sync.on('error', function (err) { - throw new Error('Failed to sync with S3: ' + err); -}); - -sync.on('fileUploadEnd', function () {uploaded++;}); - -sync.on('end', function () { - process.stdout.write(util.format( - 'Uploaded %d local files. Removed %d remote files.\n', - uploaded, sync.deleteTotal - )); - process.exit(); -}); diff --git a/package.json b/package.json index 9f77dde74..6f922e101 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,6 @@ "react-redux": "4.4.0", "react-slick": "0.9.2", "redux-thunk": "2.0.1", - "s3": "4.4.0", "sass-lint": "1.5.1", "sass-loader": "2.0.1", "scratchr2_translations": "git://github.com/LLK/scratchr2_translations.git#master", diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..82d75c412 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +s3cmd==1.6.1 From fce7c0cd9befe7321e0aeb09396afc5dcb8e161d Mon Sep 17 00:00:00 2001 From: Ray Schamp Date: Fri, 29 Apr 2016 15:40:44 -0400 Subject: [PATCH 2/2] Purge all after configuring Fastly --- README.md | 15 ++++++++------- bin/configure-fastly.js | 4 ++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9d1fd9cfa..58e9a780f 100644 --- a/README.md +++ b/README.md @@ -70,13 +70,14 @@ pip install -r requirements.txt make deploy ``` -| Variable | Default | Description | -| ----------------------- | ------- | ----------------------------------------------- | -| `FASTLY_SERVICE_ID` | `''` | Fastly service ID for `bin/configure-fastly.js` | -| `FASTLY_API_KEY` | `''` | Fastly API key for `bin/configure-fastly.js` | -| `AWS_ACCESS_KEY_ID` | `''` | AWS access key id for S3 | -| `AWS_SECRET_ACCESS_KEY` | `''` | AWS secret access key for S3 | -| `S3_BUCKET_NAME` | `''` | S3 bucket name to deploy into | +| Variable | Default | Description | +| ------------------------ | ------- | ------------------------------------------------ | +| `FASTLY_SERVICE_ID` | `''` | Fastly service ID for `bin/configure-fastly.js` | +| `FASTLY_API_KEY` | `''` | Fastly API key for `bin/configure-fastly.js` | +| `FASTLY_ACTIVATE_CHANGES`| `false` | Activate changes and purge all after configuring | +| `AWS_ACCESS_KEY_ID` | `''` | AWS access key id for S3 | +| `AWS_SECRET_ACCESS_KEY` | `''` | AWS secret access key for S3 | +| `S3_BUCKET_NAME` | `''` | S3 bucket name to deploy into | ### Current issues with the development diff --git a/bin/configure-fastly.js b/bin/configure-fastly.js index e45c2d303..8366960be 100644 --- a/bin/configure-fastly.js +++ b/bin/configure-fastly.js @@ -255,6 +255,10 @@ async.auto({ fastly.activateVersion(results.version, function (err, response) { if (err) throw new Error(err); process.stdout.write('Successfully configured and activated version ' + response.number + '\n'); + fastly.purgeAll(FASTLY_SERVICE_ID, function (err) { + if (err) throw new Error(err); + process.stdout.write('Purged all.\n'); + }); }); } }