Verifier now runs all solutions and lets you choose which campaigns and languages to skip.

app/models/Level.coffee

@@ -258,3 +258,14 @@ module.exports = class Level extends CocoModel
     else
       options.url = "/db/course/#{courseID}/levels/#{levelOriginalID}/next"
     @fetch(options)
+
+  getSolutions: ->
+    return [] unless hero = _.find (@get("thangs") ? []), id: 'Hero Placeholder'
+    return [] unless config = _.find(hero.components ? [], (x) -> x.config?.programmableMethods?.plan)?.config
+    solutions = _.cloneDeep config.programmableMethods.plan.solutions ? []
+    for solution in solutions
+      try
+        solution.source = _.template(solution.source)(config?.programmableMethods?.plan.context)
+      catch e
+        console.error "Problem with template and solution comments for", @get('slug'), e
+    solutions

app/templates/editor/verifier/verifier-view.jade

@@ -2,80 +2,101 @@ extends /templates/base-flat
 
 block content
   .container
-    div.row(style="margin-top: 20px")
+    div.row.verifier-row
       div.col-sm-3
-        p.alert.alert-success(style="padding: 5px")
+        p.alert.alert-success
           | Passed: #{view.passed}
       div.col-sm-3
-        p.alert.alert-warning(style="padding: 5px")
+        p.alert.alert-warning
           | Test Problem: #{view.problem}
       div.col-sm-3
-        p.alert.alert-danger(style="padding: 5px")
+        p.alert.alert-danger
           | Failed: #{view.failed}
       div.col-sm-3
-        p.alert.alert-info(style="padding: 5px")
+        p.alert.alert-info
           | To Run: #{view.testCount - view.passed - view.problem - view.failed}
 
-    if view.levelIDs 
+    if view.levelsByCampaign
+      .form.form-inline
+        .row
+          each campaignInfo, campaign in view.levelsByCampaign
+            .form-group.campaign-mix
+              - var campaignID = "campaign-" + campaign + "-checkbox";
+              input(id=campaignID, type="checkbox", checked=campaignInfo.checked, disabled=!!view.tests)
+              label(for=campaignID)= campaign + ': ' + campaignInfo.levels.length
+        .row
+          each codeLanguage in view.codeLanguages
+            .form-group.code-language-mix
+              - var codeLanguageID = "code-language-" + codeLanguage.id + "-checkbox";
+              input(id=codeLanguageID, type="checkbox", checked=codeLanguage.checked, disabled=!!view.tests)
+              label(for=codeLanguageID)= codeLanguage.id
+          .pull-right
+            button.btn.btn-primary#go-button(disabled=!!view.tests) Start Tests
+
+    if view.levelsToLoad && !view.tests
       .progress
-        .progress-bar.progress-bar-success(role="progressbar" style="width: #{100*view.passed/view.testCount}%")
+        .progress-bar.progress-bar-success(role="progressbar" style="width: #{100*(1 - view.levelsToLoad/view.initialLevelsToLoad)}%")
-        .progress-bar.progress-bar-warning(role="progressbar" style="width: #{100*view.problem/view.testCount}%")
-        .progress-bar.progress-bar-danger(role="progressbar" style="width: #{100*view.failed/view.testCount}%")
 
+    if view.tests
+      if view.levelIDs
+        .progress
+          .progress-bar.progress-bar-success(role="progressbar" style="width: #{100*view.passed/view.testCount}%")
+          .progress-bar.progress-bar-warning(role="progressbar" style="width: #{100*view.problem/view.testCount}%")
+          .progress-bar.progress-bar-danger(role="progressbar" style="width: #{100*view.failed/view.testCount}%")
 
-    each test, id in view.tests
+      each test, id in view.tests
-      - if (test.state == 'no-solution')
+        - if (test.state == 'no-solution')
-      -   continue;
+        -   continue;
-      if test.level
+        if test.level
-        .pull-right
+          .pull-right
-          - var last = test.level.get('slug') + view.linksQueryString 
+            - var last = test.level.get('slug') + view.linksQueryString
-          a.btn.btn-primary(href="/editor/verifier/" + last) Focus
+            a.btn.btn-primary(href="/editor/verifier/" + last) Focus
-          a.btn.btn-success(href="/play/level/" + last) Play
+            a.btn.btn-success(href="/play/level/" + last) Play
-          a.btn.btn-warning(href="/editor/level/" + last) Edit
+            a.btn.btn-warning(href="/editor/level/" + last) Edit
-          a.btn.btn-default(data-target='#verifier-test-' + id, data-toggle="collapse") Toggle
+            a.btn.btn-default(data-target='#verifier-test-' + id, data-toggle="collapse") Toggle
 
-        if !test.goals
+          if !test.goals
-          h2(style='color: orange')= test.level.get('name')
+            h2.test-running= test.level.get('name')
-            small= ' in ' + test.language + ''
+              small= ' in ' + test.language + ''
-        else if test.isSuccessful()
+          else if test.isSuccessful()
-          h2(style='color: green')= test.level.get('name')
+            h2.test-success= test.level.get('name')
-            small= ' in ' + test.language + ''
+              small= ' in ' + test.language + ''
-        else
+          else
-          h2(style='color: red')= test.level.get('name')
+            h2.test-failed= test.level.get('name')
-            small= ' in ' + test.language + ''  
+              small= ' in ' + test.language + ''
 
-        div.row(class=(test.isSuccessful() && id > 1 ? 'collapse' : 'collapse in'), id='verifier-test-' + id)
+          div.row(class=(test.isSuccessful() && id > 1 ? 'collapse' : 'collapse in'), id='verifier-test-' + id)
-          div.col-xs-8
+            div.col-xs-8
-            if test.solution
+              if test.solution
-              pre #{test.solution.source}
+                pre #{test.solution.source}
-            else
-              h4 Error Loading Test
-              pre #{test.error}
-          div.col-xs-4.well
-            if test.goals
-              if test.frames == test.solution.frameCount
-                div(style='color: green') ✓ Frames: #{test.frames}
               else
-                div(style='color: red') ✘ Frames: #{test.frames} vs #{test.solution.frameCount}
+                h4 Error Loading Test
-
+                pre #{test.error}
-              each v,k in test.goals || []
+            div.col-xs-4.well
-                if !test.solution.goals
+              if test.goals
-                  div(style='color: orange') ? #{k} (#{v.status})
+                if test.frames == test.solution.frameCount
-                else if v.status == test.solution.goals[k]
+                  div.test-success ✓ Frames: #{test.frames}
-                  div(style='color: green') ✓ #{k} (#{v.status})
                 else
-                  div(style='color: red') ✘ #{k} (#{v.status} vs #{test.solution.goals[k]})
+                  div.test-failed ✘ Frames: #{test.frames} vs #{test.solution.frameCount}
-            else
-              h3 Pending....
 
-            if test.error
+                each v,k in test.goals || []
-                pre(style="color: red") #{test.error}
+                  if !test.solution.goals
-          
+                    div.test-running ? #{k} (#{v.status})
-            if test.userCodeProblems.length
+                  else if v.status == test.solution.goals[k]
-              h4(style="color: red") User Code Problems
+                    div.test-success ✓ #{k} (#{v.status})
-              pre(style="color: red") #{JSON.stringify(test.userCodeProblems, null, 2)}
+                  else
+                    div.test-failed ✘ #{k} (#{v.status} vs #{test.solution.goals[k]})
+              else
+                h3 Pending....
 
-      else
+              if test.error
-        h1 Loading Level...
+                  pre.test-faile #{test.error}
 
-      // TODO: show last frame hash
+              if test.userCodeProblems.length
+                h4.test-failed User Code Problems
+                pre.test-failed #{JSON.stringify(test.userCodeProblems, null, 2)}
+
+        else
+          h1 Loading Level...
+
+        // TODO: show last frame hash

app/views/editor/verifier/VerifierTest.coffee

@@ -42,15 +42,8 @@ module.exports = class VerifierTest extends CocoClass
     @register()
 
   configureSession: (session, level) =>
-    # TODO: reach into and find hero and get the config from the solution
     try
-      hero = _.find level.get("thangs"), id: "Hero Placeholder"
+      session.solution = _.find level.getSolutions(), language: session.get('codeLanguage')
-      config = _.find(hero.components, (x) -> x.config?.programmableMethods?.plan).config
-      programmable = config.programmableMethods.plan
-      solution = _.find (programmable.solutions ? []), language: session.get('codeLanguage')
-      solution.source = _.template(solution.source)(config?.programmableMethods?.plan.context)
-      session.solution = solution
-
       session.set 'heroConfig', session.solution.heroConfig
       session.set 'code', {'hero-placeholder': plan: session.solution.source}
       state = session.get 'state'

app/views/editor/verifier/VerifierView.coffee

@@ -4,66 +4,102 @@ RootView = require 'views/core/RootView'
 template = require 'templates/editor/verifier/verifier-view'
 VerifierTest = require './VerifierTest'
 SuperModel = require 'models/SuperModel'
+Campaigns = require 'collections/Campaigns'
+Level = require 'models/Level'
 
 module.exports = class VerifierView extends RootView
   className: 'style-flat'
   template: template
   id: 'verifier-view'
 
+  events:
+    'click #go-button': 'onClickGoButton'
+
   constructor: (options, @levelID) ->
     super options
     # TODO: sort tests by unexpected result first
     @passed = 0
     @failed = 0
     @problem = 0
+    @testCount = 0
 
-    testLevels = [
+    if @levelID
-      'dungeons-of-kithgard', 'gems-in-the-deep', 'shadow-guard', 'kounter-kithwise', 'crawlways-of-kithgard',
+      @levelIDs = [@levelID]
-      'enemy-mine', 'illusory-interruption', 'forgetful-gemsmith', 'signs-and-portents', 'favorable-odds',
+      @testLanguages = ['python', 'javascript', 'java', 'lua', 'coffeescript']
-      'true-names', 'the-prisoner', 'banefire', 'the-raised-sword', 'kithgard-librarian', 'fire-dancing',
+      @startTestingLevels()
-      'loop-da-loop', 'haunted-kithmaze', 'riddling-kithmaze', 'descending-further', 'the-second-kithmaze',
+    else
-      'dread-door', 'cupboards-of-kithgard', 'hack-and-dash', 'known-enemy', 'master-of-names', 'lowly-kithmen',
+      @campaigns = new Campaigns()
-      'closing-the-distance', 'tactical-strike', 'the-skeleton', 'a-mayhem-of-munchkins', 'the-final-kithmaze',
+      @supermodel.trackRequest @campaigns.fetch(data: {project: 'slug,type,levels'})
-      'the-gauntlet', 'radiant-aura', 'kithgard-gates', 'destroying-angel', 'deadly-dungeon-rescue',
+      @campaigns.comparator = (m) ->
-      'breakout', 'attack-wisely', 'kithgard-mastery', 'kithgard-apprentice', 'robot-ragnarok',
+        ['intro', 'course-2', 'course-3', 'course-4', 'course-5', 'course-6', 'course-8',
-      'defense-of-plainswood', 'peasant-protection', 'forest-fire-dancing', 'course-winding-trail',
+         'dungeon', 'forest', 'desert', 'mountain', 'glacier', 'volcano'].indexOf(m.get('slug'))
-      'patrol-buster', 'endangered-burl', 'thumb-biter', 'gems-or-death', 'village-guard', 'thornbush-farm',
-      'back-to-back', 'ogre-encampment', 'woodland-cleaver', 'shield-rush', 'range-finder', 'munchkin-swarm',
-      'stillness-in-motion', 'the-agrippa-defense', 'backwoods-bombardier', 'coinucopia', 'copper-meadows',
-      'drop-the-flag', 'mind-the-trap', 'signal-corpse', 'rich-forager',
 
-      'the-mighty-sand-yak', 'oasis', 'sarven-road', 'sarven-gaps', 'thunderhooves', 'minesweeper',
+  onLoaded: ->
-      'medical-attention', 'sarven-sentry', 'keeping-time', 'hoarding-gold', 'decoy-drill', 'continuous-alchemy',
+    super()
-      'dust', 'desert-combat', 'sarven-savior', 'lurkers', 'preferential-treatment', 'sarven-shepherd',
+    return if @levelID
-      'shine-getter',
+    @filterCampaigns()
+    @filterCodeLanguages()
+    @render()
 
-      'a-fine-mint', 'borrowed-sword', 'cloudrip-commander', 'crag-tag',
+  filterCampaigns: ->
-      'hunters-and-prey', 'hunting-party',
+    @levelsByCampaign = {}
-      'leave-it-to-cleaver', 'library-tactician', 'mad-maxer', 'mad-maxer-strikes-back',
+    for campaign in @campaigns.models when campaign.get('type') in ['course', 'hero'] and campaign.get('slug') isnt 'picoctf'
-      'mirage-maker', 'mixed-unit-tactics', 'mountain-mercenaries',
+      @levelsByCampaign[campaign.get('slug')] ?= {levels: [], checked: true}
-      'noble-sacrifice', 'odd-sandstorm', 'ogre-gorge-gouger', 'reaping-fire',
+      campaignInfo = @levelsByCampaign[campaign.get('slug')]
-      'return-to-thornbush-farm', 'ring-bearer', 'sand-snakes',
+      for levelID, level of campaign.get('levels') when level.type not in ['hero-ladder', 'course-ladder', 'game-dev']
-      'slalom', 'steelclaw-gap', 'the-geometry-of-flowers',
+        campaignInfo.levels.push level.slug
-      'the-two-flowers', 'timber-guard', 'toil-and-trouble', 'village-rover',
-      'vital-powers', 'zoo-keeper',
-    ]
 
+  filterCodeLanguages: ->
+    defaultLanguages = utils.getQueryVariable('languages', 'python,javascript').split(/, ?/)
+    @codeLanguages ?= ({id: c, checked: c in defaultLanguages} for c in ['python', 'javascript', 'java', 'lua', 'coffeescript'])
+
+  onClickGoButton: (e) ->
+    @filterCampaigns()
+    @levelIDs = []
+    for campaign, campaignInfo of @levelsByCampaign
+      if @$("#campaign-#{campaign}-checkbox").is(':checked')
+        for level in campaignInfo.levels
+          @levelIDs.push level unless level in @levelIDs
+      else
+        campaignInfo.checked = false
+    @testLanguages = []
+    for codeLanguage in @codeLanguages
+      if @$("#code-language-#{codeLanguage.id}-checkbox").is(':checked')
+        codeLanguage.checked = true
+        @testLanguages.push codeLanguage.id
+      else
+        codeLanguage.checked = false
+    @startTestingLevels()
+
+  startTestingLevels: ->
+    @levelsToLoad = @initialLevelsToLoad = @levelIDs.length
+    for levelID in @levelIDs
+      level = @supermodel.getModel(Level, levelID) or new Level _id: levelID
+      if level.loaded
+        @onLevelLoaded()
+      else
+        @listenToOnce @supermodel.loadModel(level).model, 'sync', @onLevelLoaded
+
+  onLevelLoaded: (level) ->
+    if --@levelsToLoad is 0
+      @onTestLevelsLoaded()
+    else
+      @render()
+
+  onTestLevelsLoaded: ->
     defaultCores = 2
     cores = Math.max(window.navigator.hardwareConcurrency, defaultCores)
 
-    #testLevels = testLevels.slice 0, 15
     @linksQueryString = window.location.search
-    @levelIDs = if @levelID then [@levelID] else testLevels
-    languages = utils.getQueryVariable 'languages', 'python,javascript'
     #supermodel = if @levelID then @supermodel else undefined
     @tests = []
-    @taskList = []
+    @tasksList = []
-    @tasksList = _.flatten _.map @levelIDs, (v) ->
+    for levelID in @levelIDs
-      # TODO: offer good interface for choosing which languages, better performance for skipping missing solutions
+      level = @supermodel.getModel(Level, levelID)
-      #_.map ['python', 'javascript', 'coffeescript', 'lua'], (l) ->
+      solutions = level?.getSolutions()
-      _.map languages.split(','), (l) ->
+      for codeLanguage in @testLanguages
-      #_.map ['javascript'], (l) ->
+        if not solutions or _.find(solutions, language: codeLanguage)
-        level: v, language: l
+          @tasksList.push level: levelID, language: codeLanguage
 
     @testCount = @tasksList.length
     chunks = _.groupBy @tasksList, (v,i) -> i%cores