Implement headless verifier; fix headless client

app/assets/javascripts/workers/worker_world.js

@@ -312,6 +312,7 @@ self.setupDebugWorldToRunUntilFrame = function (args) {
             self.debugWorld = new World(args.userCodeMap);
             self.debugWorld.levelSessionIDs = args.levelSessionIDs;
             self.debugWorld.submissionCount = args.submissionCount;
+            self.debugWorld.fixedSeed = args.fixedSeed;
             self.debugWorld.flagHistory = args.flagHistory;
             self.debugWorld.difficulty = args.difficulty;
             if (args.level)
@@ -373,6 +374,7 @@ self.runWorld = function runWorld(args) {
     self.world = new World(args.userCodeMap);
     self.world.levelSessionIDs = args.levelSessionIDs;
     self.world.submissionCount = args.submissionCount;
+    self.world.fixedSeed = args.fixedSeed;
     self.world.flagHistory = args.flagHistory || [];
     self.world.difficulty = args.difficulty || 0;
     if(args.level)
@@ -412,15 +414,17 @@ self.onWorldLoaded = function onWorldLoaded() {
     self.goalManager.worldGenerationEnded();
   var goalStates = self.goalManager.getGoalStates();
   var overallStatus = self.goalManager.checkOverallStatus();
-  if(self.world.ended)
-    self.postMessage({type: 'end-load-frames', goalStates: goalStates, overallStatus: overallStatus});
+  var totalFrames = self.world.totalFrames;
+  if(self.world.ended) {
+    var lastFrameHash = self.world.frames[totalFrames - 2].hash
+    self.postMessage({type: 'end-load-frames', goalStates: goalStates, overallStatus: overallStatus, totalFrames: totalFrames, lastFrameHash: lastFrameHash});
+  }
   var t1 = new Date();
   var diff = t1 - self.t0;
   if(self.world.headless)
     return console.log('Headless simulation completed in ' + diff + 'ms.');
 
   var worldEnded = self.world.ended;
-  var totalFrames = self.world.totalFrames;
   var transferableSupported = self.transferableSupported();
   try {
     var serialized = self.world.serialize();

app/core/Router.coffee

@@ -87,6 +87,8 @@ module.exports = class CocoRouter extends Backbone.Router
     'editor/poll': go('editor/poll/PollSearchView')
     'editor/poll/:articleID': go('editor/poll/PollEditView')
     'editor/thang-tasks': go('editor/ThangTasksView')
+    'editor/verifier': go('editor/verifier/VerifierView')
+    'editor/verifier/:levelID': go('editor/verifier/VerifierView')
 
     'file/*path': 'routeToServer'
 

app/lib/Angel.coffee

@@ -82,7 +82,7 @@ module.exports = class Angel extends CocoClass
         clearTimeout @condemnTimeout
       when 'end-load-frames'
         clearTimeout @condemnTimeout
-        @beholdGoalStates event.data.goalStates, event.data.overallStatus  # Work ends here if we're headless.
+        @beholdGoalStates event.data.goalStates, event.data.overallStatus, false, event.data.totalFrames, event.data.lastFrameHash  # Work ends here if we're headless.
       when 'end-preload-frames'
         clearTimeout @condemnTimeout
         @beholdGoalStates event.data.goalStates, event.data.overallStatus, true
@@ -125,10 +125,13 @@ module.exports = class Angel extends CocoClass
       else
         @log 'Received unsupported message:', event.data
 
-  beholdGoalStates: (goalStates, overallStatus, preload=false) ->
+  beholdGoalStates: (goalStates, overallStatus, preload=false, totalFrames=undefined, lastFrameHash=undefined) ->
     return if @aborting
-    Backbone.Mediator.publish 'god:goals-calculated', goalStates: goalStates, preload: preload, overallStatus: overallStatus, god: @shared.god
-    @shared.god.trigger 'goals-calculated', goalStates: goalStates, preload: preload, overallStatus: overallStatus
+    event = goalStates: goalStates, preload: preload, overallStatus: overallStatus, god: @shared.god
+    event.totalFrames = totalFrames if totalFrames?
+    event.lastFrameHash = lastFrameHash if lastFrameHash?
+    Backbone.Mediator.publish 'god:goals-calculated', event
+    @shared.god.trigger 'goals-calculated', event
     @finishWork() if @shared.headless
 
   beholdWorld: (serialized, goalStates, startFrame, endFrame, streamingWorld) ->
@@ -274,15 +277,16 @@ module.exports = class Angel extends CocoClass
   simulateSync: (work) =>
     console?.profile? "World Generation #{(Math.random() * 1000).toFixed(0)}" if imitateIE9?
     work.t0 = now()
-    work.testWorld = testWorld = new World work.userCodeMap
-    work.testWorld.levelSessionIDs = work.levelSessionIDs
-    work.testWorld.submissionCount = work.submissionCount
-    work.testWorld.flagHistory = work.flagHistory ? []
-    work.testWorld.difficulty = work.difficulty
-    testWorld.loadFromLevel work.level
-    work.testWorld.preloading = work.preload
-    work.testWorld.headless = work.headless
-    work.testWorld.realTime = work.realTime
+    work.world = testWorld = new World work.userCodeMap
+    work.world.levelSessionIDs = work.levelSessionIDs
+    work.world.submissionCount = work.submissionCount
+    work.world.fixedSeed = work.fixedSeed
+    work.world.flagHistory = work.flagHistory ? []
+    work.world.difficulty = work.difficulty
+    work.world.loadFromLevel work.level
+    work.world.preloading = work.preload
+    work.world.headless = work.headless
+    work.world.realTime = work.realTime
     if @shared.goalManager
       testGM = new GoalManager(testWorld)
       testGM.setGoals work.goals
@@ -295,8 +299,13 @@ module.exports = class Angel extends CocoClass
 
     # If performance was really a priority in IE9, we would rework things to be able to skip this step.
     goalStates = testGM?.getGoalStates()
-    work.testWorld.goalManager.worldGenerationEnded() if work.testWorld.ended
-    serialized = testWorld.serialize()
+    work.world.goalManager.worldGenerationEnded() if work.world.ended
+
+    if work.headless
+      @beholdGoalStates goalStates, testGM.checkOverallStatus(), false, work.world.totalFrames, work.world.frames[work.world.totalFrames - 2]?.hash
+      return
+
+    serialized = world.serialize()
     window.BOX2D_ENABLED = false
     World.deserialize serialized.serializedWorld, @shared.worldClassMap, @shared.lastSerializedWorldFrames, @finishBeholdingWorld(goalStates), serialized.startFrame, serialized.endFrame, work.level
     window.BOX2D_ENABLED = true
@@ -304,14 +313,14 @@ module.exports = class Angel extends CocoClass
 
   doSimulateWorld: (work) ->
     work.t1 = now()
-    Math.random = work.testWorld.rand.randf  # so user code is predictable
+    Math.random = work.world.rand.randf  # so user code is predictable
     Aether.replaceBuiltin('Math', Math)
     replacedLoDash = _.runInContext(window)
     _[key] = replacedLoDash[key] for key, val of replacedLoDash
     i = 0
-    while i < work.testWorld.totalFrames
-      frame = work.testWorld.getFrame i++
+    while i < work.world.totalFrames
+      frame = work.world.getFrame i++
     Backbone.Mediator.publish 'god:world-load-progress-changed', progress: 1, god: @shared.god
-    work.testWorld.ended = true
-    system.finish work.testWorld.thangs for system in work.testWorld.systems
+    work.world.ended = true
+    system.finish work.world.thangs for system in work.world.systems
     work.t2 = now()

app/lib/God.coffee

@@ -64,6 +64,7 @@ module.exports = class God extends CocoClass
   onTomeCast: (e) ->
     return unless e.god is @
     @lastSubmissionCount = e.submissionCount
+    @lastFixedSeed = e.fixedSeed
     @lastFlagHistory = (flag for flag in e.flagHistory when flag.source isnt 'code')
     @lastDifficulty = e.difficulty
     @createWorld e.spells, e.preload, e.realTime
@@ -94,6 +95,7 @@ module.exports = class God extends CocoClass
       level: @level
       levelSessionIDs: @levelSessionIDs
       submissionCount: @lastSubmissionCount
+      fixedSeed: @lastFixedSeed
       flagHistory: @lastFlagHistory
       difficulty: @lastDifficulty
       goals: @angelsShare.goalManager?.getGoals()
@@ -126,6 +128,7 @@ module.exports = class God extends CocoClass
         level: @level
         levelSessionIDs: @levelSessionIDs
         submissionCount: @lastSubmissionCount
+        fixedSeed: @fixedSeed
         flagHistory: @lastFlagHistory
         difficulty: @lastDifficulty
         goals: @goalManager?.getGoals()

app/lib/LevelBus.coffee

@@ -247,6 +247,7 @@ module.exports = class LevelBus extends Bus
     return if _.isEmpty @changedSessionProperties
     # don't let peeking admins mess with the session accidentally
     return unless @session.get('multiplayer') or @session.get('creator') is me.id
+    return if @session.fake
     Backbone.Mediator.publish 'level:session-will-save', session: @session
     patch = {}
     patch[prop] = @session.get(prop) for prop of @changedSessionProperties

app/lib/LevelLoader.coffee

@@ -33,6 +33,7 @@ module.exports = class LevelLoader extends CocoClass
     @team = options.team
     @headless = options.headless
     @sessionless = options.sessionless
+    @fakeSessionConfig = options.fakeSessionConfig
     @spectateMode = options.spectateMode ? false
     @observing = options.observing
     @courseID = options.courseID
@@ -68,11 +69,46 @@ module.exports = class LevelLoader extends CocoClass
       @supermodel.addRequestResource(url: '/picoctf/problems', success: (picoCTFProblems) =>
         @level?.picoCTFProblem = _.find picoCTFProblems, pid: @level.get('picoCTFProblem')
       ).load()
-    @loadSession() unless @sessionless
+    if @sessionless
+      null
+    else if @fakeSessionConfig?
+      @loadFakeSession()
+    else
+      @loadSession()
     @populateLevel()
 
   # Session Loading
 
+  loadFakeSession: ->
+    if @level.get('type', true) in ['hero', 'hero-ladder', 'hero-coop']
+      @sessionDependenciesRegistered = {}
+    initVals =
+      level:
+        original: @level.get('original')
+        majorVersion: @level.get('version').major
+      creator: me.id
+      state:
+        complete: false
+        scripts: {}
+      permissions: [
+        {target: me.id, access: 'owner'}
+        {target: 'public', access: 'write'}
+      ]
+      codeLanguage: @fakeSessionConfig.codeLanguage or me.get('aceConfig')?.language or 'python'
+      _id: 'A Fake Session ID'
+    @session = new LevelSession initVals
+    @session.loaded = true
+    @fakeSessionConfig.callback? @session, @level
+
+    # TODO: set the team if we need to, for multiplayer
+    # TODO: just finish the part where we make the submit button do what is right when we are fake
+    # TODO: anything else to make teacher session-less play make sense when we are fake
+    # TODO: make sure we are not actually calling extra save/patch/put things throwing warnings because we know we are fake and so we shouldn't try to do that
+    for method in ['save', 'patch', 'put']
+      @session[method] = -> console.error "We shouldn't be doing a session.#{method}, since it's a fake session."
+    @session.fake = true
+    @loadDependenciesForSession @session
+
   loadSession: ->
     if @level.get('type', true) in ['hero', 'hero-ladder', 'hero-coop']
       @sessionDependenciesRegistered = {}
@@ -171,7 +207,7 @@ module.exports = class LevelLoader extends CocoClass
     browser['platform'] = $.browser.platform if $.browser.platform
     browser['version'] = $.browser.version if $.browser.version
     session.set 'browser', browser
-    session.patch()
+    session.patch() unless session.fake
 
   consolidateFlagHistory: ->
     state = @session.get('state') ? {}

app/models/SuperModel.coffee

@@ -192,8 +192,8 @@ module.exports = class SuperModel extends Backbone.Model
     return res
 
   checkName: (name) ->
-    if _.isString(name)
-      console.warn("SuperModel name property deprecated. Remove '#{name}' from code.")
+    #if _.isString(name)
+    #  console.warn("SuperModel name property deprecated. Remove '#{name}' from code.")
 
   storeResource: (resource, value) ->
     @rid++

app/schemas/subscriptions/god.coffee

@@ -49,6 +49,8 @@ module.exports =
     goalStates: goalStatesSchema
     preload: {type: 'boolean'}
     overallStatus: {type: ['string', 'null'], enum: ['success', 'failure', 'incomplete', null]}
+    totalFrames: {type: ['integer', 'undefined']}
+    lastFrameHash: {type: ['number', 'undefined']}
 
   'god:world-load-progress-changed': c.object {required: ['progress', 'god']},
     god: {type: 'object'}

app/schemas/subscriptions/misc.coffee

@@ -70,3 +70,6 @@ module.exports =
 
   'application:service-loaded': c.object {required: ['service']},
     service: {type: 'string'}  # 'segment'
+
+  'test:update': c.object {},
+     state: {type: 'string'}

app/schemas/subscriptions/tome.coffee

@@ -12,6 +12,7 @@ module.exports =
     preload: {type: 'boolean'}
     realTime: {type: 'boolean'}
     submissionCount: {type: 'integer'}
+    fixedSeed: {type: ['integer', 'undefined']}
     flagHistory: {type: 'array'}
     difficulty: {type: 'integer'}
     god: {type: 'object'}

app/templates/editor/verifier/verifier-view.jade

@@ -0,0 +1,33 @@
+extends /templates/base-flat
+
+block content
+  .container
+    each test in view.tests
+      if test.level
+        h2= test.level.get('name')
+          small= ' in ' + test.language + ''
+        div.well(style='width: 300px; float: right')
+          if test.goals
+            each v,k in test.goals || []
+              case v.status
+                when 'success': div(style='color: green') ✓ #{k}
+                when 'incomplete': div(style='color: orange') ✘ #{k}
+                when 'failure': div(style='color: red') ✘ #{k}
+                default: div(style='color: blue') #{k}
+          else
+            h3 Running....
+        if test.solution
+          pre(style='margin-right: 350px') #{test.solution.source}
+        else
+          h4 Solution not found...
+      else
+        h1 Loading Level...
+  
+      div#tome-view
+      div#goals-veiw
+  
+      br
+
+      // TODO: show errors
+      // TODO: frame length
+      // TODO: show last frame hash

app/views/editor/verifier/VerifierTest.coffee

@@ -0,0 +1,102 @@
+CocoClass = require 'core/CocoClass'
+SuperModel = require 'models/SuperModel'
+{createAetherOptions} = require 'lib/aether_utils'
+God = require 'lib/God'
+GoalManager = require 'lib/world/GoalManager'
+LevelLoader = require 'lib/LevelLoader'
+
+module.exports = class VerifierTest extends CocoClass
+  constructor: (@levelID, @updateCallback, @supermodel, @language) ->
+    super()
+    # TODO: turn this into a Subview
+    # TODO: listen to Backbone.Mediator.publish 'god:non-user-code-problem', problem: event.data.problem, god: @shared.god from Angel to detect when we can't load the thing
+    # TODO: listen to the progress report from Angel to show a simulation progress bar (maybe even out of the number of frames we actually know it'll take)
+    @supermodel ?= new SuperModel()
+    @language ?= 'python'
+    @load()
+
+  load: ->
+    @loadStartTime = new Date()
+    @god = new God maxAngels: 1, headless: true
+    @levelLoader = new LevelLoader supermodel: @supermodel, levelID: @levelID, headless: true, fakeSessionConfig: {codeLanguage: @language, callback: @configureSession}
+    @listenToOnce @levelLoader, 'world-necessities-loaded', @onWorldNecessitiesLoaded
+
+  onWorldNecessitiesLoaded: ->
+    # Called when we have enough to build the world, but not everything is loaded
+    @grabLevelLoaderData()
+
+    unless @solution
+      @updateCallback? state: 'error'
+      @error = 'No solution present...'
+      @state = 'error'
+      return
+    me.team = @team = 'humans'
+    @setupGod()
+    @initGoalManager()
+    @register()
+
+  configureSession: (session, level) =>
+    # TODO: reach into and find hero and get the config from the solution
+    try
+      hero = _.find level.get("thangs"), id: "Hero Placeholder"
+      programmable = _.find(hero.components, (x) -> x.config?.programmableMethods?.plan).config.programmableMethods.plan
+      session.solution = _.find (programmable.solutions ? []), language: session.get('codeLanguage')
+      session.set 'heroConfig', session.solution.heroConfig
+      session.set 'code', {'hero-placeholder': plan: session.solution.source}
+      state = session.get 'state'
+      state.flagHistory = session.solution.flagHistory
+      state.difficulty = session.solution.difficulty or 0
+      session.solution.seed = undefined unless _.isNumber session.solution.seed  # TODO: migrate away from submissionCount/sessionID seed objects
+    catch e
+      @state = 'error'
+      @error = "Could not load the session solution for #{level.get('name')}: " + e.toString()
+
+  grabLevelLoaderData: ->
+    @world = @levelLoader.world
+    @level = @levelLoader.level
+    @session = @levelLoader.session
+    @solution = @levelLoader.session.solution
+
+  setupGod: ->
+    @god.setLevel @level.serialize @supermodel, @session
+    @god.setLevelSessionIDs [@session.id]
+    @god.setWorldClassMap @world.classMap
+    @god.lastFlagHistory = @session.get('state').flagHistory
+    @god.lastDifficulty = @session.get('state').difficulty
+    @god.lastFixedSeed = @session.solution.seed
+    @god.lastSubmissionCount = 0
+
+  initGoalManager: ->
+    @goalManager = new GoalManager(@world, @level.get('goals'), @team)
+    @god.setGoalManager @goalManager
+
+  register: ->
+    @listenToOnce @god, 'infinite-loop', @fail  # TODO: have one of these
+
+    @listenToOnce @god, 'goals-calculated', @processSingleGameResults
+    @god.createWorld @generateSpellsObject()
+    @updateCallback? state: 'running'
+
+  processSingleGameResults: (e) ->
+    @goals = e.goalStates
+    @frames = e.totalFrames
+    @lastFrameHash = e.lastFrameHash
+    @state = 'complete'
+    @updateCallback? state: @state
+
+  fail: (e) ->
+    @error = 'Failed due to infinate loop.'
+    @state = 'error'
+    @updateCallback? state: @state 
+
+  generateSpellsObject: ->
+    aetherOptions = createAetherOptions functionName: 'plan', codeLanguage: @session.get('codeLanguage')
+    spellThang = aether: new Aether aetherOptions
+    spells = "hero-placeholder/plan": thangs: {'Hero Placeholder': spellThang}, name: 'plan'
+    source = @session.get('code')['hero-placeholder'].plan
+    try
+      spellThang.aether.transpile source
+    catch e
+      console.log "Couldn't transpile!\n#{source}\n", e
+      spellThang.aether.transpile ''
+    spells

app/views/editor/verifier/VerifierView.coffee

@@ -0,0 +1,35 @@
+RootView = require 'views/core/RootView'
+template = require 'templates/editor/verifier/verifier-view'
+VerifierTest = require './VerifierTest'
+
+module.exports = class VerifierView extends RootView
+  className: 'style-flat'
+  template: template
+  id: 'verifier-view'
+  events:
+    'input input': 'searchUpdate'
+    'change input': 'searchUpdate'
+
+  constructor: (options, @levelID) ->
+    super options
+    # TODO: rework to handle N at a time instead of all at once
+    # TODO: sort tests by unexpected result first
+    testLevels = ["dungeons-of-kithgard", "gems-in-the-deep", "shadow-guard", "kounter-kithwise", "crawlways-of-kithgard", "enemy-mine", "illusory-interruption", "forgetful-gemsmith", "signs-and-portents", "favorable-odds", "true-names", "the-prisoner", "banefire", "the-raised-sword", "kithgard-librarian", "fire-dancing", "loop-da-loop", "haunted-kithmaze", "riddling-kithmaze", "descending-further", "the-second-kithmaze", "dread-door", "cupboards-of-kithgard", "hack-and-dash", "known-enemy", "master-of-names", "lowly-kithmen", "closing-the-distance", "tactical-strike", "the-skeleton", "a-mayhem-of-munchkins", "the-final-kithmaze", "the-gauntlet", "radiant-aura", "kithgard-gates", "destroying-angel", "deadly-dungeon-rescue", "kithgard-brawl", "cavern-survival", "breakout", "attack-wisely", "kithgard-mastery", "kithgard-apprentice", "robot-ragnarok", "defense-of-plainswood", "peasant-protection", "forest-fire-dancing"]
+    #testLevels = testLevels.slice 0, 15
+    levelIDs = if @levelID then [@levelID] else testLevels
+    #supermodel = if @levelID then @supermodel else undefined
+    @tests = []
+    async.eachSeries levelIDs, (levelID, lnext) =>
+      async.eachSeries ['python','javascript'], (lang, next) =>
+        @tests.unshift new VerifierTest levelID, (e) =>
+          @update(e)
+          next() if e.state in ['complete', 'error']
+        , @supermodel, lang
+      , -> lnext()
+
+  update: (event) =>
+    # TODO: show unworkable tests instead of hiding them
+    # TODO: destroy them Tests after or something
+    console.log 'got event', event, 'on some test'
+    @tests = _.filter @tests, (test) -> test.state isnt 'error'
+    @render()

app/views/play/level/PlayLevelView.coffee

@@ -135,7 +135,7 @@ module.exports = class PlayLevelView extends RootView
 
   load: ->
     @loadStartTime = new Date()
-    @god = new God debugWorker: true
+    @god = new God()
     @levelLoader = new LevelLoader supermodel: @supermodel, levelID: @levelID, sessionID: @sessionID, opponentSessionID: @opponentSessionID, team: @getQueryVariable('team'), observing: @observing, courseID: @courseID
     @listenToOnce @levelLoader, 'world-necessities-loaded', @onWorldNecessitiesLoaded
 

app/views/play/level/tome/TomeView.coffee

@@ -169,7 +169,7 @@ module.exports = class TomeView extends CocoView
     difficulty = sessionState.difficulty ? 0
     if @options.observing
       difficulty = Math.max 0, difficulty - 1  # Show the difficulty they won, not the next one.
-    Backbone.Mediator.publish 'tome:cast-spells', spells: @spells, preload: preload, realTime: realTime, submissionCount: sessionState.submissionCount ? 0, flagHistory: sessionState.flagHistory ? [], difficulty: difficulty, god: @options.god
+    Backbone.Mediator.publish 'tome:cast-spells', spells: @spells, preload: preload, realTime: realTime, submissionCount: sessionState.submissionCount ? 0, flagHistory: sessionState.flagHistory ? [], difficulty: difficulty, god: @options.god, fixedSeed: @options.fixedSeed
 
   onToggleSpellList: (e) ->
     @spellList?.rerenderEntries()

headless_client.coffee

@@ -29,7 +29,7 @@ options =
   simulateOnlyOneGame: simulateOneGame
 
 options.heapdump = require('heapdump') if options.heapdump
-server = if options.testing then 'http://127.0.0.1:3000' else 'https://codecombat.com'
+server = if options.testing then 'http://127.0.0.1:3000' else 'http://direct.codecombat.com'
 # Use direct instead of live site because jQlone's requests proxy doesn't do caching properly and CloudFlare gets too aggressive.
 
 # Disabled modules
@@ -43,22 +43,25 @@ disable = [
 
 # Global emulated stuff
 GLOBAL.window = GLOBAL
-GLOBAL.document = location: pathname: 'headless_client'
+GLOBAL.document =
+  location:
+    pathname: 'headless_client'
+    search: 'esper=1'
 GLOBAL.console.debug = console.log
+GLOBAL.serverConfig =
+  picoCTF: false
+  production: false
 try
   GLOBAL.Worker = require('webworker-threads').Worker
+  Worker::removeEventListener = (what) ->
+    if what is 'message'
+      @onmessage = -> #This webworker api has only one event listener at a time.
 catch
-  console.log ""
-  console.log "Headless client needs the webworker-threads package from NPM to function."
-  console.log "Try installing it with the command:"
-  console.log ""
-  console.log "    npm install webworker-threads"
-  console.log ""
-  process.exit(1)
+  # Fall back to IE compatibility mode where it runs synchronously with no web worker.
+  # (Which we will be doing now always because webworker-threads doesn't run in newer node versions.)
+  eval require('fs').readFileSync('./vendor/scripts/Box2dWeb-2.1.a.3.js', 'utf8')
+  GLOBAL.Box2D = Box2D
 
-Worker::removeEventListener = (what) ->
-  if what is 'message'
-    @onmessage = -> #This webworker api has only one event listener at a time.
 GLOBAL.tv4 = require('tv4').tv4
 GLOBAL.TreemaUtils = require bowerComponentsPath + 'treema/treema-utils'
 GLOBAL.marked = setOptions: ->

headless_client/cluster.coffee

@@ -0,0 +1,63 @@
+child_process = require 'child_process'
+chalk = require 'chalk'
+_ = require 'lodash'
+Promise = require 'bluebird'
+path = require 'path'
+
+cores = 4
+
+list = [
+  "dungeons-of-kithgard", "gems-in-the-deep", "shadow-guard", "kounter-kithwise", "crawlways-of-kithgard",
+  "enemy-mine", "illusory-interruption", "forgetful-gemsmith", "signs-and-portents", "favorable-odds",
+  "true-names", "the-prisoner", "banefire", "the-raised-sword", "kithgard-librarian", "fire-dancing",
+  "loop-da-loop", "haunted-kithmaze", "riddling-kithmaze", "descending-further", "the-second-kithmaze",
+  "dread-door", "cupboards-of-kithgard", "hack-and-dash", "known-enemy", "master-of-names",
+  "lowly-kithmen", "closing-the-distance", "tactical-strike", "the-skeleton", "a-mayhem-of-munchkins",
+  "the-final-kithmaze", "the-gauntlet", "radiant-aura", "kithgard-gates", "destroying-angel", "deadly-dungeon-rescue",
+  "kithgard-brawl", "cavern-survival", "breakout", "attack-wisely", "kithgard-mastery", "kithgard-apprentice", 
+  "robot-ragnarok", "defense-of-plainswood", "peasant-protection", "forest-fire-dancing"
+]
+
+c1 = ["dungeons-of-kithgard", "gems-in-the-deep", "shadow-guard", "enemy-mine", "true-names", "fire-dancing", "loop-da-loop", "haunted-kithmaze", "the-second-kithmaze", "dread-door", "cupboards-of-kithgard", "breakout", "known-enemy", "master-of-names", "a-mayhem-of-munchkins", "the-gauntlet", "the-final-kithmaze", "kithgard-gates", "wakka-maul"]
+c2 = ["defense-of-plainswood", "course-winding-trail", "patrol-buster", "endangered-burl", "thumb-biter", "gems-or-death", "village-guard", "thornbush-farm", "back-to-back", "ogre-encampment", "woodland-cleaver", "shield-rush", "range-finder", "peasant-protection", "munchkin-swarm", "forest-fire-dancing", "stillness-in-motion", "the-agrippa-defense", "backwoods-bombardier", "coinucopia", "copper-meadows", "drop-the-flag", "mind-the-trap", "signal-corpse", "rich-forager", "cross-bones"]
+
+list = [].concat(c1, c2)
+list = c1
+
+list = _.shuffle(list);
+
+lpad = (s, l, color = 'white') ->
+  return chalk[color](s.substring(0, l)) if s.length >= l
+  return chalk[color](s + new Array(l - s.length).join(' '))
+
+
+chunks = _.groupBy list, (v,i) -> i%cores
+_.forEach chunks, (list, cid) ->
+  console.log(list)
+  cp = child_process.fork path.join(__dirname, './verifier.js'), list, silent: true
+  cp.on 'message', (m) ->
+    return if m.state is 'running'
+    okay = true
+    goals = _.map m.observed.goals, (v,k) ->
+      return lpad('No Goals Set', 15, 'yellow') unless m.solution.goals
+      lpad(k, 15, if v == m.solution.goals[k] then 'green' else 'red')
+
+
+    extra = []
+    if m.observed.frameCount == m.solution.frameCount
+      extra.push lpad('F:' + m.observed.frameCount, 15, 'green')
+    else
+      extra.push lpad('F:' + m.observed.frameCount  + ' vs ' + m.solution.frameCount , 15, 'red')
+      okay = false
+
+    if m.observed.lastHash == m.solution.lastHash
+      extra.push lpad('Hash', 5, 'green')
+    else
+      extra.push lpad('Hash' , 5, 'red')
+      okay = false
+
+    col = if okay then 'green' else 'red'
+    if m.state is 'error' or m.error 
+      console.log lpad(m.level, 30, 'red') + lpad(m.language, 15, 'cyan') + chalk.red(m.error)
+    else
+      console.log lpad(m.level, 30, col) + lpad(m.language, 15, 'cyan') + ' ' + extra.join(' ') + ' ' + goals.join(' ') 

headless_client/jQlone.coffee

@@ -8,8 +8,6 @@ module.exports = $ = (input) ->
   append: (input)-> exports: ()->
 
 # Non-standard jQuery stuff. Don't use outside of server.
-$._debug = false
-$._server = 'https://codecombat.com'
 $._cookies = request.jar()
 
 $.when = Deferred.when

headless_client/verifier.js

@@ -0,0 +1,3 @@
+require('coffee-script');
+require('coffee-script/register');
+var server = require('../verifier.coffee');

package.json

@@ -58,6 +58,7 @@
     "aws-sdk": "~2.0.0",
     "bayesian-battle": "0.0.7",
     "bluebird": "^3.2.1",
+    "chalk": "^1.1.3",
     "co-express": "^1.2.1",
     "coffee-script": "1.9.x",
     "connect": "2.7.x",

verifier.coffee

@@ -0,0 +1,147 @@
+useEsper = false
+bowerComponentsPath = './bower_components/'
+headlessClientPath = './headless_client/'
+
+# SETTINGS
+options =
+  workerCode: require headlessClientPath + 'worker_world'
+  debug: false # Enable logging of ajax calls mainly
+  testing: false # Instead of simulating 'real' games, use the same one over and over again. Good for leak hunting.
+  testFile: require headlessClientPath + 'test.js'
+  leakTest: false # Install callback that tries to find leaks automatically
+  exitOnLeak: false # Exit if leak is found. Only useful if leaktest is set to true, obviously.
+  heapdump: false # Dumps the whole heap after every pass. The heap dumps can then be viewed in Chrome browser.
+  headlessClient: true
+
+options.heapdump = require('heapdump') if options.heapdump
+server = if options.testing then 'http://127.0.0.1:3000' else 'http://direct.codecombat.com'
+# Use direct instead of live site because jQlone's requests proxy doesn't do caching properly and CloudFlare gets too aggressive.
+
+# Disabled modules
+disable = [
+  'lib/AudioPlayer'
+  'locale/locale'
+  '../locale/locale'
+]
+
+# Start of the actual code. Setting up the enivronment to match the environment of the browser
+
+# Global emulated stuff
+GLOBAL.window = GLOBAL
+GLOBAL.document =
+  location:
+    pathname: 'headless_client'
+    search: ''
+
+GLOBAL.console.debug = console.log
+GLOBAL.serverConfig =
+  picoCTF: false
+  production: false
+
+#try
+#  GLOBAL.Worker = require('webworker-threads').Worker
+#catch e
+#  GLOBAL.Worker = require('./headless_client/fork_web_worker').Worker
+#  options.workerCode = './worker_world.coffee'
+#
+#Worker::removeEventListener = (what) ->
+#  if what is 'message'
+#    @onmessage = -> #This webworker api has only one event listener at a time.
+GLOBAL.tv4 = require('tv4').tv4
+GLOBAL.TreemaUtils = require bowerComponentsPath + 'treema/treema-utils'
+GLOBAL.marked = setOptions: ->
+store = {}
+GLOBAL.localStorage =
+    getItem: (key) => store[key]
+    setItem: (key, s) => store[key] = s
+    removeItem: (key) => delete store[key]
+GLOBAL.lscache = require bowerComponentsPath + 'lscache/lscache'
+GLOBAL.esper = require bowerComponentsPath + 'esper.js/esper'
+
+# Hook node.js require. See https://github.com/mfncooper/mockery/blob/master/mockery.js
+# The signature of this function *must* match that of Node's Module._load,
+# since it will replace that.
+# (Why is there no easier way?)
+# the path used for the loader. __dirname is module dependent.
+path = __dirname
+m = require 'module'
+originalLoader = m._load
+hookedLoader = (request, parent, isMain) ->
+  if request in disable or ~request.indexOf('templates')
+    console.log 'Ignored ' + request if options.debug
+    return class fake
+  else if /node_modules[\\\/]aether[\\\/]/.test parent.id
+    null  # Let it through
+  else if '/' in request and not (request[0] is '.') or request is 'application'
+    #console.log 'making path', path + '/app/' + request, 'from', path, request, 'with parent', parent
+    request = path + '/app/' + request
+  else if request is 'underscore'
+    request = 'lodash'
+  console.log 'loading ' + request if options.debug
+  originalLoader request, parent, isMain
+
+unhook = () ->
+  m._load = originalLoader
+hook = () ->
+  m._load = hookedLoader
+
+GLOBAL.$ = GLOBAL.jQuery = require headlessClientPath + 'jQlone'
+$._debug = options.debug
+$._server = server
+
+do (setupLodash = this) ->
+  GLOBAL._ = require 'lodash'
+  _.str = require 'underscore.string'
+  _.string = _.str
+  _.mixin _.str.exports()
+
+# load Backbone. Needs hooked loader to reroute underscore to lodash.
+hook()
+GLOBAL.Backbone = require bowerComponentsPath + 'backbone/backbone'
+# Use original loader for theese
+unhook()
+Backbone.$ = $
+require bowerComponentsPath + 'validated-backbone-mediator/backbone-mediator'
+Backbone.Mediator.setValidationEnabled false
+GLOBAL.Aether = require 'aether'
+eval require('fs').readFileSync('./vendor/scripts/Box2dWeb-2.1.a.3.js', 'utf8')
+GLOBAL.Box2D = Box2D
+# Set up new loader. Again.
+hook()
+
+
+SuperModel = require 'models/SuperModel'
+VerifierTest = require('views/editor/verifier/VerifierTest')
+
+supermodel = new SuperModel()
+
+oldGetQueryVariable = require('core/utils').getQueryVariable
+require('core/utils').getQueryVariable = (args...) ->
+  return useEsper if args[0] is 'esper'
+  oldGetQueryVariable args...
+
+list = process.argv.slice(2);
+async = require 'async'
+
+
+
+async.eachSeries list, (item, next) ->
+  async.eachSeries ['python','javascript'], (lang, lnext) ->
+    test = new VerifierTest item, (e) ->
+      return if e.state is 'running'
+      obj =
+        error: test.error
+        state: e.state
+        level: item,
+        language: lang
+        observed:
+          goals: _.mapValues(test.goals, 'status')
+          frameCount: test.frames
+          lastHash: test.lastFrameHash
+        solution:
+          test.solution
+      process.send?(obj)
+      console.log(obj)
+      lnext() if e.state in ['error','complete']
+    , supermodel, lang
+  , () -> next()