Update school counts page with geoip and 10+ buckets

Placing teachers and students in unknown districts and schools if we
have geoip countries and states/regions.
Placing teachers/students in unknown schools if there are 10+ students.

app/templates/admin/school-counts.jade

@@ -14,8 +14,9 @@ block content
     p
       div Untriaged students: #{view.untriagedStudents}
       div Untriaged teachers: #{view.untriagedTeachers}
-    .small Teacher: owns a classroom or has a teacher role
+    .small Teacher: teacherish role or owns a classroom
-    .small Student: member of a classroom or has schoolName set, not in HoC course instance
+    .small Student: student role or member of a classroom or has schoolName set, not in HoC course instance
+    .small School: trial request data or teacher with 10+ students
     .small +3 USA states are GU, PR, DC
 
     p

app/views/admin/SchoolCountsView.coffee

@@ -7,6 +7,10 @@ User = require 'models/User'
 utils = require 'core/utils'
 
 # TODO: match anonymous trial requests with real users via email
+# TODO: sanitize and use student.schoolName, can't use it directly
+# TODO: example untriaged student: no geo IP, not attached to teacher with school
+# TODO: example untriaged teacher: deleted but owner of a classroom
+# TODO: use student geoip on their teacher
 
 module.exports = class SchoolCountsView extends RootView
   id: 'admin-school-counts-view'
@@ -35,7 +39,7 @@ module.exports = class SchoolCountsView extends RootView
     studentMap = {} # Used to make sure teachers and students only counted once
     studentNonHocMap = {} # Used to exclude HoC users
     teacherStudentMap = {} # Used to link students to their teacher locations
-    countryStateDistrictSchoolCountsMap = {} # Data graph
+    unknownSchoolCount = 1 # Used to separate unique but unknown schools
 
     console.log(new Date().toISOString(), "Processing #{@courseInstances.models.length} course instances...")
     for courseInstance in @courseInstances.models
@@ -46,25 +50,29 @@ module.exports = class SchoolCountsView extends RootView
     for classroom in @classrooms.models
       teacherID = classroom.get('ownerID')
       teacherMap[teacherID] ?= {}
-      teacherMap[teacherID] = true
       teacherStudentMap[teacherID] ?= {}
       for studentID in classroom.get('members')
+        continue if teacherMap[studentID]
         continue unless studentNonHocMap[studentID]
-        studentMap[studentID] = true
+        studentMap[studentID] = {}
         teacherStudentMap[teacherID][studentID] = true
 
     console.log(new Date().toISOString(), "Processing #{@teachers.models.length} teachers...")
     for teacher in @teachers.models
-      teacherMap[teacher.id] ?= {}
+      teacherMap[teacher.id] = teacher.get('geo') ? {}
       delete studentMap[teacher.id]
 
     console.log(new Date().toISOString(), "Processing #{@students.models.length} students...")
-    for student in @students.models when not teacherMap[student.id]
+    for student in @students.models
       continue unless studentNonHocMap[student.id]
-      schoolName = student.get('schoolName')
+      continue if teacherMap[student.id]
-      studentMap[student.id] = true
+      studentMap[student.id] = {geo: student.get('geo')}
 
-    console.log(new Date().toISOString(), "Processing trial #{@trialRequests.models.length} requests...")
+    orphanStudentMap = _.cloneDeep(studentMap)
+    orphanTeacherMap = _.cloneDeep(teacherMap)
+
+    console.log(new Date().toISOString(), "Processing #{@trialRequests.models.length} trial requests...")
+    countryStateDistrictSchoolCountsMap = {}
     for trialRequest in @trialRequests.models
       teacherID = trialRequest.get('applicant')
       unless teacherMap[teacherID]
@@ -82,8 +90,10 @@ module.exports = class SchoolCountsView extends RootView
         countryStateDistrictSchoolCountsMap[country][state][district] ?= {}
         countryStateDistrictSchoolCountsMap[country][state][district][school] ?= {students: {}, teachers: {}}
         countryStateDistrictSchoolCountsMap[country][state][district][school].teachers[teacherID] = true
-        for studentID, val of teacherStudentMap[teacherID]
+        for studentID, val of teacherStudentMap[teacherID] when orphanStudentMap[studentID]
           countryStateDistrictSchoolCountsMap[country][state][district][school].students[studentID] = true
+          delete orphanStudentMap[studentID]
+        delete orphanTeacherMap[teacherID]
       else if not _.isEmpty(props.country)
         country = props.country?.trim()
         country = country[0].toUpperCase() + country.substring(1).toLowerCase()
@@ -102,8 +112,65 @@ module.exports = class SchoolCountsView extends RootView
         countryStateDistrictSchoolCountsMap[country][state][district] ?= {}
         countryStateDistrictSchoolCountsMap[country][state][district][school] ?= {students: {}, teachers: {}}
         countryStateDistrictSchoolCountsMap[country][state][district][school].teachers[teacherID] = true
-        for studentID, val of teacherStudentMap[teacherID]
+        for studentID, val of teacherStudentMap[teacherID] when orphanStudentMap[studentID]
           countryStateDistrictSchoolCountsMap[country][state][district][school].students[studentID] = true
+          delete orphanStudentMap[studentID]
+        delete orphanTeacherMap[teacherID]
+
+    console.log(new Date().toISOString(), "Processing #{Object.keys(orphanTeacherMap).length} orphaned teachers with geo IPs...")
+    for teacherID, val of orphanTeacherMap
+      continue unless teacherMap[teacherID].country
+      country = teacherMap[teacherID].countryName or teacherMap[teacherID].country
+      country = 'UK' if country is 'GB' or country is 'United Kingdom'
+      country = 'USA' if country is 'US' or country is 'United States'
+      state = teacherMap[teacherID].region or 'unknown'
+      district = 'unknown'
+      school = 'unknown'
+      if teacherStudentMap[teacherID] and Object.keys(teacherStudentMap[teacherID]).length >= 10
+        school += unknownSchoolCount++
+      countryStateDistrictSchoolCountsMap[country] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state][district] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state][district][school] ?= {students: {}, teachers: {}}
+      countryStateDistrictSchoolCountsMap[country][state][district][school].teachers[teacherID] = true
+      if teacherStudentMap[teacherID] and Object.keys(teacherStudentMap[teacherID]).length >= 10
+        for studentID, val of teacherStudentMap[teacherID] when orphanStudentMap[studentID]
+          countryStateDistrictSchoolCountsMap[country][state][district][school].students[studentID] = true
+          delete orphanStudentMap[studentID]
+      delete orphanTeacherMap[teacherID]
+
+    console.log(new Date().toISOString(), "Processing #{Object.keys(orphanTeacherMap).length} orphaned teachers with 10+ students...")
+    for teacherID, val of orphanTeacherMap
+      continue unless teacherStudentMap[teacherID] and Object.keys(teacherStudentMap[teacherID]).length >= 10
+      country = 'unknown'
+      state = 'unknown'
+      district = 'unknown'
+      school = "unknown#{unknownSchoolCount++}"
+      countryStateDistrictSchoolCountsMap[country] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state][district] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state][district][school] ?= {students: {}, teachers: {}}
+      countryStateDistrictSchoolCountsMap[country][state][district][school].teachers[teacherID] = true
+      for studentID, val of teacherStudentMap[teacherID] when orphanStudentMap[studentID]
+        countryStateDistrictSchoolCountsMap[country][state][district][school].students[studentID] = true
+        delete orphanStudentMap[studentID]
+      delete orphanTeacherMap[teacherID]
+
+    console.log(new Date().toISOString(), "Processing #{Object.keys(orphanStudentMap).length} orphaned students with geo IPs...")
+    for studentID, val of orphanStudentMap
+      continue unless studentMap[studentID].geo?.country
+      country = studentMap[studentID].geo.countryName or studentMap[studentID].geo.country
+      country = 'UK' if country is 'GB' or country is 'United Kingdom'
+      country = 'USA' if country is 'US' or country is 'United States'
+      state = studentMap[studentID].geo.region or 'unknown'
+      district = 'unknown'
+      school = 'unknown'
+      countryStateDistrictSchoolCountsMap[country] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state][district] ?= {}
+      countryStateDistrictSchoolCountsMap[country][state][district][school] ?= {students: {}, teachers: {}}
+      countryStateDistrictSchoolCountsMap[country][state][district][school].students[studentID] = true
+      delete orphanStudentMap[studentID]
 
     console.log(new Date().toISOString(), 'Building country graphs...')
     @countryGraphs = {}
@@ -146,11 +213,16 @@ module.exports = class SchoolCountsView extends RootView
         schools: @countryGraphs[country].totalSchools
         students: @countryGraphs[country].totalStudents
         teachers: @countryGraphs[country].totalTeachers
-      totalStudents += @countryGraphs[country].totalSchools
+      totalStudents += @countryGraphs[country].totalStudents
       totalTeachers += @countryGraphs[country].totalTeachers
+
+    # Compare against orphanStudentMap and orphanTeacherMap to catch bugs
     @untriagedStudents = Object.keys(studentMap).length - totalStudents
     @untriagedTeachers = Object.keys(teacherMap).length - totalTeachers
 
+    console.log(new Date().toISOString(), "teacherMap #{Object.keys(teacherMap).length} totalTeachers #{totalTeachers} orphanTeacherMap #{Object.keys(orphanTeacherMap).length}  @untriagedTeachers #{@untriagedTeachers}")
+    console.log(new Date().toISOString(), "studentMap #{Object.keys(studentMap).length} totalStudents #{totalStudents} orphanStudentMap #{Object.keys(orphanStudentMap).length}  @untriagedStudents #{@untriagedStudents}")
+
     for country, graph of @countryGraphs
       graph.stateCounts.sort (a, b) ->
         b.students - a.students or b.teachers - a.teachers or b.schools - a.schools or b.districts - a.districts or b.state.localeCompare(a.state)

package.json

@@ -63,6 +63,7 @@
     "co-express": "^1.2.1",
     "coffee-script": "1.9.x",
     "connect": "2.7.x",
+    "country-list": "0.0.3",
     "express": "~3.0.6",
     "express-useragent": "~0.0.9",
     "geoip-lite": "^1.1.6",
@@ -101,7 +102,6 @@
     "commonjs-require-definition": "0.2.0",
     "compressible": "~1.0.1",
     "country-data": "0.0.24",
-    "country-list": "0.0.3",
     "css-brunch": "^1.7.0",
     "fs-extra": "^0.26.2",
     "http-proxy": "^1.13.2",

server/middleware/users.coffee

@@ -1,6 +1,8 @@
 _ = require 'lodash'
 co = require 'co'
+countryList = require('country-list')()
 errors = require '../commons/errors'
+geoip = require 'geoip-lite'
 wrap = require 'co-express'
 Promise = require 'bluebird'
 parse = require '../commons/parse'
@@ -10,7 +12,6 @@ sendwithus = require '../sendwithus'
 User = require '../models/User'
 Classroom = require '../models/Classroom'
 
-
 module.exports =
   fetchByGPlusID: wrap (req, res, next) ->
     gpID = req.query.gplusID
@@ -97,11 +98,22 @@ module.exports =
 
   getStudents: wrap (req, res, next) ->
     throw new errors.Unauthorized('You must be an administrator.') unless req.user?.isAdmin()
-    students = yield User.find({$and: [{schoolName: {$exists: true}}, {schoolName: {$ne: ''}}, {anonymous: false}]}).select('schoolName').lean()
+    query = $or: [{role: 'student'}, {$and: [{schoolName: {$exists: true}}, {schoolName: {$ne: ''}}, {anonymous: false}]}]
-    res.status(200).send(students)
+    users = yield User.find(query).select('lastIP schoolName').lean()
+    for user in users
+      if ip = user.lastIP
+        user.geo = geoip.lookup(ip)
+        if country = user.geo?.country
+          user.geo.countryName = countryList.getName(country)
+    res.status(200).send(users)
 
   getTeachers: wrap (req, res, next) ->
     throw new errors.Unauthorized('You must be an administrator.') unless req.user?.isAdmin()
     teacherRoles = ['teacher', 'technology coordinator', 'advisor', 'principal', 'superintendent', 'parent']
-    teachers = yield User.find(anonymous: false, role: {$in: teacherRoles}).select('').lean()
+    users = yield User.find(anonymous: false, role: {$in: teacherRoles}).select('lastIP').lean()
-    res.status(200).send(teachers)
+    for user in users
+      if ip = user.lastIP
+        user.geo = geoip.lookup(ip)
+        if country = user.geo?.country
+          user.geo.countryName = countryList.getName(country)
+    res.status(200).send(users)