refactor: Merge changes from upstream and resolve issues with interface and test coverage

Merges in changes from upstream and resolves issues with both the removal of the analysis library as
well as issues with lint rules and integration tests.
This commit is contained in:
Andrew Sliwinski 2018-03-24 11:36:17 -04:00
commit 0fe264a05b
16 changed files with 638 additions and 59 deletions

View file

@ -5,17 +5,27 @@ var parse = require('./lib/parse');
var validate = require('./lib/validate');
/**
* Unpacks, parses, validates, and analyzes Scratch projects. If successful,
* will return a valid Scratch project object with appended metadata.
* @param {Buffer | string} input Buffer or string representing project
* @param {Function} callback Returns error or project data
*/
* Unpacks, parses, validates, and analyzes Scratch projects. If successful,
* will return a valid Scratch project object with appended metadata.
* @param {Buffer | string} input Buffer or string representing project
* @param {Function} callback Returns error or project data
*/
module.exports = function (input, callback) {
async.waterfall([
function (cb) {
unpack(input, cb);
},
parse,
validate
], callback);
// First unpack the input (need this outside of the async waterfall so that
// unpackedProject can be refered to again)
unpack(input, function (err, unpackedProject) {
if (err) return callback(err);
async.waterfall([
function (cb) {
parse(unpackedProject[0], cb);
},
validate
], function (error, validatedInput) {
// One more callback wrapper so that we can re-package everything
// with the possible zip returned from unpack
if (error) return callback(error);
callback(null, [validatedInput, unpackedProject[1]]);
});
});
};

363
lib/sb3_schema.json Normal file
View file

@ -0,0 +1,363 @@
{
"$id": "https://scratch.mit.edu/sb3_schema.json",
"$schema": "http://json-schema.org/schema#",
"description": "Scratch 3.0 Project Schema",
"definitions": {
"optionalString": {
"oneOf": [
{"type": "string"},
{"type": "null"}
]
},
"boolOrBoolString": {
"oneOf": [
{"type": "string",
"enum": ["true", "false"]},
{"type": "boolean"}
]
},
"assetId": {
"type": "string",
"pattern": "^[a-fA-F0-9]{32}$"
},
"costume": {
"type": "object",
"properties": {
"assetId": { "$ref": "#/definitions/assetId"},
"bitmapResolution": {
"type": "integer"
},
"dataFormat": {
"type": "string",
"enum": ["png", "PNG", "svg", "SVG", "jpeg", "JPEG", "jpg", "JPG", "bmp", "BMP"]
},
"md5ext": {
"type": "string",
"pattern": "^[a-fA-F0-9]{32}.[a-zA-Z]+$"
},
"name": {
"type": "string"
},
"rotationCenterX": {
"type": "number"
},
"rotationCenterY": {
"type": "number"
}
},
"required": [
"assetId",
"dataFormat",
"name",
"rotationCenterX",
"rotationCenterY"
]
},
"sound": {
"type": "object",
"properties": {
"assetId": { "$ref": "#/definitions/assetId"},
"dataFormat": {
"type": "string",
"enum": ["wav", "WAV", "wave", "WAVE", "mp3", "MP3"]
},
"format": {
"type": "string",
"enum": ["", "adpcm"]
},
"md5ext": {
"type": "string",
"pattern": "^[a-fA-F0-9]{32}.[a-zA-Z]+$"
},
"name": {
"type": "string"
},
"rate": {
"type": "integer"
},
"sampleCount": {
"type": "integer"
}
},
"required": [
"assetId",
"dataFormat",
"format",
"name"
]
},
"scalar_variable": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": ["", "scalar"]
},
"value": {
"oneOf": [
{"type": "string"},
{"type": "number"}
]
}
}
},
"list": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": ["list"]
},
"value": {
"type": "array"
}
}
},
"broadcast_message": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": ["broadcast_msg"]
},
"value": {
"type": "string",
"not": {"enum": [""]}
}
}
},
"block": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"opcode": {
"type": "string"
},
"inputs": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"block": {"$ref":"#/definitions/optionalString"},
"shadow": {"$ref":"#/definitions/optionalString"},
"value": {
"oneOf": [
{"type": "string"},
{"type": "number"}
]
}
}
}
},
"fields": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"value": {
"oneOf": [
{"type": "string"},
{"type": "number"}
]
}
}
}
},
"next": {"$ref":"#/definitions/optionalString"},
"topLevel": {
"type": "boolean"
},
"parent": {"$ref":"#/definitions/optionalString"},
"shadow": {
"type": "boolean"
},
"x": {
"type": "number"
},
"y": {
"type": "number"
},
"mutation": {
"type": "object",
"properties": {
"tagName": {
"type": "string",
"enum": ["mutation"]
},
"children": {
"type": "array"
},
"proccode": {
"type": "string"
},
"argumentids": {
"type": "string"
},
"warp": {"$ref":"#/definitions/boolOrBoolString"},
"hasnext": {"$ref":"#/definitions/boolOrBoolString"}
}
}
},
"required": [
"id",
"opcode"
]
},
"stage": {
"type": "object",
"description": "Description of property (and/or property/value pairs) that are unique to the stage.",
"properties": {
"name": {
"type": "string",
"enum": ["Stage"]
},
"isStage": {
"type": "boolean",
"enum": [true]
}
}
},
"sprite": {
"type": "object",
"description": "Description of property (and/or property/value pairs) for sprites.",
"properties": {
"name": {
"type": "string",
"not": {"enum": ["Stage", "stage"]}
},
"isStage": {
"type": "boolean",
"enum": [false]
},
"visible": {
"type": "boolean"
},
"x": {
"type": "number"
},
"y": {
"type": "number"
},
"size": {
"type": "number"
},
"direction": {
"type": "number"
},
"draggable": {
"type": "boolean"
},
"rotationStyle": {
"type": "string",
"enum": ["all around", "don't rotate", "left-right"]
}
},
"required": [
"name",
"isStage"
]
},
"target": {
"type": "object",
"description" : "Properties common to both Scratch 3.0 Stage and Sprite",
"properties": {
"currentCostume": {
"type": "integer",
"minimum": 0
},
"blocks": {
"type": "object",
"additionalProperties": {"$ref":"#/definitions/block"}
},
"variables": {
"type": "object",
"additionalProperties": {
"oneOf": [
{"$ref":"#/definitions/scalar_variable"},
{"$ref":"#/definitions/list"},
{"$ref":"#/definitions/broadcast_message"}
]
}
},
"costumes": {
"type": "array",
"items": {"$ref":"#/definitions/costume"},
"minItems": 1,
"uniqueItems": true
},
"sounds":{
"type": "array",
"items": {"$ref":"#/definitions/sound"},
"uniqueItems": true
}
},
"required": [
"variables",
"costumes",
"sounds",
"blocks"
]
}
},
"type": "object",
"properties": {
"meta": {
"type": "object",
"properties": {
"semver": {
"type": "string",
"pattern": "^(3.[0-9]+.[0-9]+)$"
},
"vm": {
"type": "string",
"pattern": "^([0-9]+.[0-9]+.[0-9]+)($|-)"
},
"agent": {
"type": "string"
}
},
"required": [
"semver"
]
},
"targets": {
"type": "array",
"items": [
{
"allOf": [
{"$ref": "#/definitions/stage" },
{"$ref": "#/definitions/target"}
]
}
],
"additionalItems": {
"allOf": [
{"$ref": "#/definitions/sprite"},
{"$ref": "#/definitions/target"}
]
}
}
},
"required": [
"meta",
"targets"
]
}

View file

@ -1,4 +1,4 @@
var JSZip = require('jszip');
var unzip = require('./unzip');
/**
* If input a buffer, transforms buffer into a UTF-8 string.
@ -11,13 +11,17 @@ var JSZip = require('jszip');
module.exports = function (input, callback) {
if (typeof input === 'string') {
// Pass string to callback
return callback(null, input);
return callback(null, [input, null]);
}
// Validate input type
var typeError = 'Input must be a Buffer or a string.';
if (!Buffer.isBuffer(input)) {
return callback(typeError);
try {
input = new Buffer(input);
} catch (e) {
return callback(typeError);
}
}
// Determine format
@ -33,17 +37,12 @@ module.exports = function (input, callback) {
if (signature.indexOf('80 75') === 0) isZip = true;
// If not legacy or zip, convert buffer to UTF-8 string and return
if (!isZip && !isLegacy) return callback(null, input.toString('utf-8'));
if (!isZip && !isLegacy) {
return callback(null, [input.toString('utf-8'), null]);
}
// Return error if legacy encoding detected
if (isLegacy) return callback('Parser only supports Scratch 2.X');
// Handle zip
// @todo Handle error
JSZip.loadAsync(input).then(function (zip) {
zip.file('project.json').async('string')
.then(function (project) {
callback(null, project);
});
});
unzip(input, callback);
};

21
lib/unzip.js Normal file
View file

@ -0,0 +1,21 @@
var JSZip = require('jszip');
/**
* Unpacks a zip file.
* @param {string} input Zip file provided as a string
* @param {array} callback Array including both the project and zip archive
* @return {void}
*/
module.exports = function (input, callback) {
return JSZip.loadAsync(input)
.then(function (zip) {
return zip.file('project.json').async('string')
.then(function (project) {
return callback(null, [project, zip]);
});
})
.catch(function (err) {
var msg = 'Failed to unzip and extract project.json, with error: ';
return callback(msg + JSON.stringify(err));
});
};

View file

@ -1,10 +1,29 @@
var ajv = require('ajv')();
var schema = require('./schema.json');
var sb2schema = require('./sb2_schema.json');
var sb3schema = require('./sb3_schema.json');
module.exports = function (input, callback) {
var validate = ajv.compile(schema);
var valid = validate(input);
var validateSb2 = ajv.compile(sb2schema);
var validateSb3 = ajv.compile(sb3schema);
if (!valid) return callback(validate.errors);
callback(null, input);
var isValidSb2 = validateSb2(input);
if (isValidSb2) {
input.projectVersion = 2;
return callback(null, input);
}
var isValidSb3 = validateSb3(input);
if (isValidSb3) {
input.projectVersion = 3;
return callback(null, input);
}
var validationErrors = {
validationError: 'Could not parse as a valid SB2 or SB3 project.',
sb2Errors: validateSb2.errors,
sb3Errors: validateSb3.errors
};
callback(validationErrors);
};

View file

@ -40,5 +40,6 @@
"commitizen": {
"path": "./node_modules/cz-conventional-changelog"
}
}
},
"version": "1.0.0"
}

Binary file not shown.

BIN
test/fixtures/data/_zipNoProjectJson.zip vendored Normal file

Binary file not shown.

View file

@ -1,4 +1,5 @@
var test = require('tap').test;
var JSZip = require('jszip');
var data = require('../fixtures/data');
var parser = require('../../index');
@ -11,28 +12,40 @@ test('sb', function (t) {
});
test('sb2', function (t) {
parser(data.empty.sb2, function (err, res) {
parser(data.empty.sb2, function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip instanceof JSZip, true);
t.end();
});
});
test('json', function (t) {
parser(data.empty.json, function (err, res) {
parser(data.empty.json, function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip, null);
t.end();
});
});
test('json string', function (t) {
parser(data.empty.json.toString('utf-8'), function (err, res) {
parser(data.empty.json.toString('utf-8'), function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip, null);
t.end();
});
});

View file

@ -1,4 +1,5 @@
var test = require('tap').test;
var JSZip = require('jszip');
var data = require('../fixtures/data');
var parser = require('../../index');
@ -11,28 +12,40 @@ test('sb', function (t) {
});
test('sb2', function (t) {
parser(data.example.sb2, function (err, res) {
parser(data.example.sb2, function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip instanceof JSZip, true);
t.end();
});
});
test('json', function (t) {
parser(data.example.json, function (err, res) {
parser(data.example.json, function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip, null);
t.end();
});
});
test('json string', function (t) {
parser(data.example.json.toString('utf-8'), function (err, res) {
parser(data.example.json.toString('utf-8'), function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip, null);
t.end();
});
});

View file

@ -1,4 +1,5 @@
var test = require('tap').test;
var JSZip = require('jszip');
var data = require('../fixtures/data');
var parser = require('../../index');
@ -15,36 +16,48 @@ test('sb', function (t) {
test('sb2', function (t) {
var set = data.sb2;
t.plan(set.length * 3);
t.plan(set.length * 5);
for (var i in data.sb2) {
parser(data.sb2[i], function (err, res) {
parser(data.sb2[i], function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip instanceof JSZip, true);
});
}
});
test('json', function (t) {
var set = data.json;
t.plan(set.length * 3);
t.plan(set.length * 5);
for (var i in data.json) {
parser(data.json[i], function (err, res) {
parser(data.json[i], function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip, null);
});
}
});
test('json string', function (t) {
var set = data.json;
t.plan(set.length * 3);
t.plan(set.length * 5);
for (var i in data.json) {
parser(data.json[i].toString('utf-8'), function (err, res) {
parser(data.json[i].toString('utf-8'), function (err, result) {
t.equal(err, null);
t.equal(Array.isArray(result), true);
var res = result[0];
var possibleZip = result[1];
t.type(res, 'object');
t.type(res.info, 'object');
t.equal(possibleZip, null);
});
}
});

View file

@ -1,7 +1,7 @@
var ajv = require('ajv')();
var test = require('tap').test;
var meta = require('../fixtures/meta.json');
var schema = require('../../lib/schema.json');
var schema = require('../../lib/sb2_schema.json');
test('spec', function (t) {
t.type(schema, 'object');

View file

@ -1,6 +1,7 @@
var fs = require('fs');
var path = require('path');
var test = require('tap').test;
var JSZip = require('jszip');
var unpack = require('../../lib/unpack');
var fixtures = {
@ -31,10 +32,12 @@ test('sb2', function (t) {
var buffer = new Buffer(fixtures.sb2);
unpack(buffer, function (err, res) {
t.equal(err, null);
t.type(res, 'string');
t.equal(Array.isArray(res), true);
t.type(res[0], 'string');
t.doesNotThrow(function () {
JSON.parse(res);
JSON.parse(res[0]);
});
t.equal(res[1] instanceof JSZip, true);
t.end();
});
});
@ -43,10 +46,12 @@ test('json', function (t) {
var buffer = new Buffer(fixtures.json);
unpack(buffer, function (err, res) {
t.equal(err, null);
t.type(res, 'string');
t.equal(Array.isArray(res), true);
t.type(res[0], 'string');
t.doesNotThrow(function () {
JSON.parse(res);
JSON.parse(res[0]);
});
t.equal(res[1], null);
t.end();
});
});
@ -55,10 +60,12 @@ test('json utf-8 string', function (t) {
var buffer = new Buffer(fixtures.json);
unpack(buffer.toString('utf-8'), function (err, res) {
t.equal(err, null);
t.type(res, 'string');
t.equal(Array.isArray(res), true);
t.type(res[0], 'string');
t.doesNotThrow(function () {
JSON.parse(res);
JSON.parse(res[0]);
});
t.equal(res[1], null);
t.end();
});
});
@ -66,10 +73,12 @@ test('json utf-8 string', function (t) {
test('invalid string', function (t) {
unpack('this is not json', function (err, res) {
t.equal(err, null);
t.type(res, 'string');
t.equal(Array.isArray(res), true);
t.type(res[0], 'string');
t.throws(function () {
JSON.parse(res);
JSON.parse(res[0]);
});
t.equal(res[1], null);
t.end();
});
});

115
test/unit/unzip.js Normal file
View file

@ -0,0 +1,115 @@
var fs = require('fs');
var path = require('path');
var test = require('tap').test;
var JSZip = require('jszip');
var unzip = require('../../lib/unzip');
var fixtures = {
sb: path.resolve(__dirname, '../fixtures/data/_example.sb'),
sb2: path.resolve(__dirname, '../fixtures/data/_example.sb2'),
zipFakeProjectJSON:
path.resolve(__dirname, '../fixtures/data/_zipFakeProjectJson.zip'),
zipNoProjectJSON:
path.resolve(__dirname, '../fixtures/data/_zipNoProjectJson.zip')
};
for (var i in fixtures) {
fixtures[i] = fs.readFileSync(fixtures[i]);
}
test('spec', function (t) {
t.type(unzip, 'function');
t.end();
});
test('sb', function (t) {
var buffer = new Buffer(fixtures.sb);
unzip(buffer, function (err, res) {
t.type(err, 'string');
var errorMessage = 'Failed to unzip and extract project.json';
t.equal(err.startsWith(errorMessage), true);
t.type(res, 'undefined');
t.end();
});
});
test('sb2', function (t) {
var buffer = new Buffer(fixtures.sb2);
unzip(buffer, function (err, res) {
t.equal(err, null);
t.equal(Array.isArray(res), true);
t.type(res[0], 'string');
t.doesNotThrow(function () {
JSON.parse(res[0]);
});
t.equal(res[1] instanceof JSZip, true);
t.end();
});
});
test('zip without project json', function (t) {
var buffer = new Buffer(fixtures.zipNoProjectJSON);
unzip(buffer, function (err, res) {
t.type(err, 'string');
var errorMessage = 'Failed to unzip and extract project.json';
t.equal(err.startsWith(errorMessage), true);
t.type(res, 'undefined');
t.end();
});
});
test('zip with fake project json', function (t) {
var buffer = new Buffer(fixtures.zipFakeProjectJSON);
unzip(buffer, function (err, res) {
t.equal(err, null);
t.equal(Array.isArray(res), true);
t.type(res[0], 'string');
t.equal(res[0], 'this is not json\n');
t.throws(function () {
JSON.parse(res[0]);
});
t.equal(res[1] instanceof JSZip, true);
t.end();
});
});
test('random string instead of zip', function (t) {
unzip('this is not a zip', function (err, res) {
t.type(err, 'string');
var errorMessage = 'Failed to unzip and extract project.json';
t.equal(err.startsWith(errorMessage), true);
t.type(res, 'undefined');
t.end();
});
});
test('undefined', function (t) {
var foo;
unzip(foo, function (err, obj) {
t.type(err, 'string');
var errorMessage = 'Failed to unzip and extract project.json';
t.equal(err.startsWith(errorMessage), true);
t.type(obj, 'undefined');
t.end();
});
});
test('null', function (t) {
unzip(null, function (err, obj) {
t.type(err, 'string');
var errorMessage = 'Failed to unzip and extract project.json';
t.equal(err.startsWith(errorMessage), true);
t.type(obj, 'undefined');
t.end();
});
});
test('object', function (t) {
unzip({}, function (err, obj) {
t.type(err, 'string');
var errorMessage = 'Failed to unzip and extract project.json';
t.equal(err.startsWith(errorMessage), true);
t.type(obj, 'undefined');
t.end();
});
});

View file

@ -17,15 +17,18 @@ test('valid', function (t) {
test('invalid', function (t) {
validate({foo: 1}, function (err, res) {
t.equal(Array.isArray(err), true);
t.type(err, 'object');
t.type(err.validationError, 'string');
var sb2Errs = err.sb2Errors;
t.equal(Array.isArray(sb2Errs), true);
t.type(res, 'undefined');
t.type(err[0], 'object');
t.type(err[0].keyword, 'string');
t.type(err[0].dataPath, 'string');
t.type(err[0].schemaPath, 'string');
t.type(err[0].message, 'string');
t.type(err[0].params, 'object');
t.type(err[0].params.missingProperty, 'string');
t.type(sb2Errs[0], 'object');
t.type(sb2Errs[0].keyword, 'string');
t.type(sb2Errs[0].dataPath, 'string');
t.type(sb2Errs[0].schemaPath, 'string');
t.type(sb2Errs[0].message, 'string');
t.type(sb2Errs[0].params, 'object');
t.type(sb2Errs[0].params.missingProperty, 'string');
t.end();
});
});