Rewrite updateCloseIoLeads.js

Moving user-specific information from customized fields to activity
notes
Lots of data reformatting and renaming
Lots of perf/readability improvements via async.parallel

scripts/updateCloseIoLeads.js

@@ -6,8 +6,25 @@ if (process.argv.length !== 5) {
   process.exit();
 }
 
-// TODO: update existing leads
+// TODO: Test multiple contacts
-// TODO: support multiple contacts per organization
+// TODO: Support multiple emails for the same contact (i.e diff trial and coco emails)
+// TODO: Update notes with new data (e.g. coco user or intercom url)
+// TODO: Find/fix case-sensitive bugs
+// TODO: Use generators and promises
+
+// Save as custom fields instead of user-specific lead notes
+const commonTrialProperties = ['organization', 'city', 'state', 'country'];
+
+// Old properties which are deprecated or moved
+const customFieldsToRemove = [
+  'coco_name', 'coco_firstName', 'coco_lastName', 'coco_gender', 'coco_numClassrooms', 'coco_numStudents', 'coco_role', 'coco_schoolName', 'coco_stats', 'coco_lastLevel',
+  'email', 'intercom_url', 'name',
+  'trial_created', 'trial_educationLevel', 'trial_phoneNumber', 'trial_email', 'trial_location', 'trial_name', 'trial_numStudents', 'trial_role', 'trial_userID', 'userID', 'trial_organization', 'trial_city', 'trial_state', 'trial_country',
+  'demo_request_organization', 'demo_request_city', 'demo_request_state', 'demo_request_country'
+];
+
+// Skip these problematic leads
+const leadsToSkip = ['6 sınıflar', 'fdsafd', 'ashtasht'];
 
 const scriptStartTime = new Date();
 const closeIoApiKey = process.argv[2];
@@ -16,38 +33,31 @@ const intercomAppId = intercomAppIdApiKey.split(':')[0];
 const intercomApiKey = intercomAppIdApiKey.split(':')[1];
 const mongoConnUrl = process.argv[4];
 const MongoClient = require('mongodb').MongoClient;
+const async = require('async');
 const request = require('request');
 
 const earliestDate = new Date();
 earliestDate.setUTCDate(earliestDate.getUTCDate() - 10);
 
-log('Finding leads..');
+// log('DEBUG: Finding leads..');
 findLeads((err, leads) => {
   if (err) {
     console.error(err);
     return;
   }
   log(`Num leads ${Object.keys(leads).length}`);
-  log('Adding Intercom data..');
+  // log('DEBUG: Adding Intercom data..');
   addIntercomData(leads, (err) => {
     if (err) {
       console.error(err);
       return;
     }
-    log('Updating leads..');
+    // log('DEBUG: Updating leads..');
-    updateLeads(leads, (err, numLeadsCreated) => {
+    updateLeads(leads, (err) => {
       if (err) {
         console.error(err);
         return;
       }
-      // // TEMP
-      // for (const email in leads) {
-      //   console.log(email);
-      //   console.log(leads[email]);
-      //   break;
-      // }
-      // // TEMP
-      log(`Num leads created ${numLeadsCreated}`);
       log("Script runtime: " + (new Date() - scriptStartTime));
     });
   });
@@ -57,72 +67,187 @@ findLeads((err, leads) => {
 /* Helpers */
 
 class Lead {
-  constructor(email) {
+  constructor(name) {
-    this.email = email;
+    this.contacts = {};
+    this.custom = {};
+    this.name = name;
   }
-  addClassroom(classroom) {
+  addClassroom(email, classroom) {
-    this.coco_numClassrooms = this.coco_numClassrooms ? this.coco_numClassrooms + 1 : 1;
+    if (!this.contacts[email.toLowerCase()]) this.contacts[email.toLowerCase()] = {};
+    const contact = this.contacts[email.toLowerCase()];
+    contact.numClassrooms = contact.numClassrooms ? contact.numClassrooms + 1 : 1;
     if (classroom.members && classroom.members.length) {
-      this.coco_numStudents = this.coco_numStudents ? this.coco_numStudents + classroom.members.length : classroom.members.length;
+      contact.numStudents = contact.numStudents ? contact.numStudents + classroom.members.length : classroom.members.length;
     }
   }
-  addIntercomUser(user) {
+  addIntercomUser(email, user) {
     if (user && user.id) {
-      this.intercom_url = `https://app.intercom.io/a/apps/${intercomAppId}/users/${user.id}/`;
+      if (!this.contacts[email.toLowerCase()]) this.contacts[email.toLowerCase()] = {};
+      this.contacts[email.toLowerCase()].intercomUrl = `https://app.intercom.io/a/apps/${intercomAppId}/users/${user.id}/`;
     }
   }
-  addTrialRequest(trialRequest) {
+  addTrialRequest(email, trial) {
-    if (trialRequest.properties) {
+    if (!this.contacts[email.toLowerCase()]) this.contacts[email.toLowerCase()] = {};
-      let location = '';
+    this.contacts[email.toLowerCase()].name = trial.properties.name;
-      if (trialRequest.properties.city) location = trialRequest.properties.city;
+    this.contacts[email.toLowerCase()].trial = trial;
-      if (trialRequest.properties.state) location += `${location ? ', ' : ''}${trialRequest.properties.state}`;
-      if (trialRequest.properties.country) location += `${location ? ', ' : ''}${trialRequest.properties.country}`;
-      if (location) this['trial_location'] = location;
-      if (trialRequest.properties.educationLevel) {
-        this['trial_educationLevel'] = trialRequest.properties.educationLevel.join(', ');
-      }
-      for (const prop in trialRequest.properties) {
-        if (['educationLevel', 'city', 'state', 'country'].indexOf(prop) >= 0) continue;
-        this[`trial_${prop}`] = trialRequest.properties[prop];
-      }
-    }
-    this.trial_created = trialRequest.created;
-    this.trial_userID = trialRequest.applicant;
-    if (this.trial_name) this.name = this.trial_name;
-    this.userID = trialRequest.applicant;
   }
-  addUser(user) {
+  addUser(email, user) {
-    // if (this.userID && !this.userID.equals(user._id)) {
+    this.contacts[email.toLowerCase()].user = user;
-    //   // console.log(`Trial request user ID mismatch for ${this.email} replacing ${this.userID} with ${user._id}`);
+  }
-    //   this.trialUserID = this.userID;
+  getLeadPostData() {
-    // }
+    const postData = {
-    this.userID = user._id;
+      display_name: this.name,
-    if (!this.name) {
+      name: this.name,
-      if (user.firstName && user.lastName) {
+      status: 'Not Attempted',
-        this.name = `${user.firstName} ${user.lastName}`;
+      contacts: this.getContactsPostData(),
+      custom: {
+        lastUpdated: new Date(),
+        'Lead Origin': 'Demo Request'
       }
-      else if (user.name) {
+    };
-        this.name = user.name;
+    for (const email in this.contacts) {
+      const props = this.contacts[email].trial.properties;
+      if (props) {
+        for (const prop in props) {
+          if (commonTrialProperties.indexOf(prop) >= 0) {
+            postData.custom[`demo_${prop}`] = props[prop];
+          }
+        }
       }
     }
-    if (user.firstName) this.coco_firstName = user.firstName;
+    return postData;
-    if (user.lastName) this.coco_lastName = user.lastName;
+  }
-    if (user.name) this.coco_name = user.name;
+  getLeadPutData(currentLead) {
-    if (user.gender) this.coco_gender = user.gender;
+    // console.log('DEBUG: getLeadPutData', currentLead.name);
-    if (user.lastLevel) this.coco_lastLevel = user.lastLevel;
+    const putData = {};
-    if (user.role) this.coco_role = user.role;
+    const currentCustom = currentLead.custom || {};
-    if (user.schoolName) this.coco_schoolName = user.schoolName;
+    if (currentCustom['Lead Origin'] !== 'Demo Request') {
-    if (user.stats) this.coco_stats = JSON.stringify(user.stats);
+      putData['custom.Lead Origin'] = 'Demo Request';
+    }
+
+    for (const email in this.contacts) {
+      const props = this.contacts[email].trial.properties;
+      if (props) {
+        for (const prop in props) {
+          if (commonTrialProperties.indexOf(prop) >= 0 && currentCustom[`demo_${prop}`] !== props[prop]) {
+            putData[`custom.demo_${prop}`] = props[prop];
+          }
+        }
+      }
+    }
+    for (const field of customFieldsToRemove) {
+      if (currentCustom[field]) {
+        putData[`custom.${field}`] = null;
+      }
+    }
+    if (Object.keys(putData).length > 0) {
+      putData[`custom.lastUpdated`] = new Date();
+    }
+    return putData;
+  }
+  getContactsPostData(existingLead) {
+    const postData = [];
+    const existingEmails = {};
+    if (existingLead) {
+      const existingContacts = existingLead.contacts || [];
+      for (const contact of existingContacts) {
+        const emails = contact.emails || [];
+        for (const email of emails) {
+          existingEmails[email.email.toLowerCase()] = true;
+        }
+      }
+    }
+    for (const email in this.contacts) {
+      if (existingEmails[email]) continue;
+      const contact = this.contacts[email];
+      const data = {
+        emails: [{email: email}],
+        name: contact.name
+      }
+      const props = contact.trial.properties;
+      if (props.phoneNumber) {
+        data.phones = [{phone: props.phoneNumber}];
+      }
+      if (props.role) {
+        data.title = props.role;
+      }
+      else if (contact.user || contact.user.role) {
+        data.title = contact.user.role;
+      }
+      postData.push(data);
+    }
+    return postData;
+  }
+  getNotesPostData(currentNotes) {
+    // Post activity notes for each contact
+    function noteExists(email) {
+      if (currentNotes) {
+        for (const note of currentNotes) {
+          if (note.note.indexOf(email) >= 0) {
+            return true;
+          }
+        }
+      }
+      return false;
+    }
+    const notes = [];
+    for (const email in this.contacts) {
+      if (!noteExists(email)) {
+        const contact = this.contacts[email];
+        let noteData = "";
+        const trial = contact.trial
+        if (trial.properties) {
+          const props = trial.properties;
+          if (props.name) {
+            noteData += `${props.name}\n`;
+          }
+          if (props.email) {
+            noteData += `demo_email: ${props.email.toLowerCase()}\n`;
+          }
+          if (trial.created) {
+            noteData += `demo_request: ${trial.created}\n`;
+          }
+          if (props.educationLevel) {
+            noteData += `demo_educationLevel: ${props.educationLevel.join(', ')}\n`;
+          }
+          for (const prop in props) {
+            if (['email', 'educationLevel', 'created'].indexOf(prop) >= 0 || commonTrialProperties.indexOf(prop) >= 0) continue;
+            noteData += `demo_${prop}: ${props[prop]}\n`;
+          }
+        }
+        if (contact.intercomUrl) noteData += `intercom_url: ${contact.intercomUrl}\n`;
+        if (contact.user) {
+          const user = contact.user
+          noteData += `coco_userID: ${user._id}\n`;
+          if (user.firstName) noteData += `coco_firstName: ${user.firstName}\n`;
+          if (user.lastName) noteData += `coco_lastName: ${user.lastName}\n`;
+          if (user.name) noteData += `coco_name: ${user.name}\n`;
+          if (user.emaillower) noteData += `coco_email: ${user.emailLower}\n`;
+          if (user.gender) noteData += `coco_gender: ${user.gender}\n`;
+          if (user.lastLevel) noteData += `coco_lastLevel: ${user.lastLevel}\n`;
+          if (user.role) noteData += `coco_role: ${user.role}\n`;
+          if (user.schoolName) noteData += `coco_schoolName: ${user.schoolName}\n`;
+          if (user.stats && user.stats.gamesCompleted) noteData += `coco_gamesCompleted: ${user.stats.gamesCompleted}\n`;
+          noteData += `coco_preferredLanguage: ${user.preferredLanguage || 'en-US'}\n`;
+        }
+        if (contact.numClassrooms) {
+          noteData += `coco_numClassrooms: ${contact.numClassrooms}\n`
+        }
+        if (contact.numStudents) {
+          noteData += `coco_numStudents: ${contact.numStudents}\n`
+        }
+        notes.push(noteData);
+      }
+    }
+    return notes;
   }
 }
 
 function findLeads(done) {
-  // Recent trial requests
-  // Recent users with teacher role
-
   MongoClient.connect(mongoConnUrl, (err, db) => {
     if (err) return done(err);
 
+    // Recent trial requests
     const query = {$and: [{created: {$gte: earliestDate}}, {type: 'course'}]};
     db.collection('trial.requests').find(query).toArray((err, trialRequests) => {
       if (err) {
@@ -130,16 +255,22 @@ function findLeads(done) {
         return done(err);
       }
       const leads = {};
+      const emailLeadMap = {};
+      const emails = [];
       for (const trialRequest of trialRequests) {
-        const email = trialRequest.properties.email;
+        if (!trialRequest.properties || !trialRequest.properties.email) continue;
-        if (!leads[email]) leads[email] = new Lead(email);
+        const email = trialRequest.properties.email.toLowerCase();
-        leads[email].addTrialRequest(trialRequest);
+        emails.push(email);
+        const name = trialRequest.properties.organization || trialRequest.properties.name || email;
+        if (!leads[name]) leads[name] = new Lead(name);
+        leads[name].addTrialRequest(email, trialRequest);
+        emailLeadMap[email] = leads[name];
       }
 
+      // Users for trial requests
       const query = {$and: [
-        {dateCreated: {$gte: earliestDate}},
+        {emailLower: {$in: emails}},
-        {anonymous: false},
+        {anonymous: false}
-        {role: {$in: ['teacher', 'technology coordinator', 'advisor', 'principal', 'superintendent']}}
       ]};
       db.collection('users').find(query).toArray((err, users) => {
         if (err) {
@@ -147,15 +278,17 @@ function findLeads(done) {
           return done(err);
         }
         const userIDs = [];
+        const userLeadMap = {};
         const userEmailMap = {};
         for (const user of users) {
           const email = user.emailLower;
-          if (!leads[email]) leads[email] = new Lead(email);
+          emailLeadMap[email].addUser(email, user);
-          leads[email].addUser(user);
           userIDs.push(user._id);
+          userLeadMap[user._id.valueOf()] = emailLeadMap[email];
           userEmailMap[user._id.valueOf()] = email;
         }
 
+        // Classrooms for users
         const query = {ownerID: {$in: userIDs}};
         db.collection('classrooms').find(query).toArray((err, classrooms) => {
           if (err) {
@@ -164,7 +297,7 @@ function findLeads(done) {
           }
 
           for (const classroom of classrooms) {
-            leads[userEmailMap[classroom.ownerID.valueOf()]].addClassroom(classroom);
+            userLeadMap[classroom.ownerID.valueOf()].addClassroom(userEmailMap[classroom.ownerID.valueOf()], classroom);
           }
           db.close();
           return done(null, leads);
@@ -174,17 +307,8 @@ function findLeads(done) {
   });
 }
 
-function addIntercomData(leads, done) {
+function createAddIntercomDataFn(lead, email) {
-  const leadEmails = Object.keys(leads);
+  return (done) => {
-  let leadIndex = 0;
-
-  function nextUser() {
-    if (leadIndex < leadEmails.length) {
-      return getUser(leadEmails[leadIndex++]);
-    }
-    return done();
-  }
-  function getUser(email) {
     const options = {
       url: `https://api.intercom.io/users?email=${encodeURIComponent(email)}`,
       auth: {
@@ -198,78 +322,172 @@ function addIntercomData(leads, done) {
     request.get(options, (error, response, body) => {
       if (error) return done(error);
       const user = JSON.parse(body);
-      leads[email].addIntercomUser(user);
+      lead.addIntercomUser(email, user);
-      return nextUser();
+      return done();
     });
-  }
+  };
-
-  getUser(leadEmails[leadIndex++]);
 }
 
-function updateLeads(leads, done) {
+function addIntercomData(leads, done) {
-  const leadEmails = Object.keys(leads);
+  const tasks = []
-  let leadIndex = 0;
+  for (const name in leads) {
-  let numLeadsCreated = 0;
+    for (const email in leads[name].contacts) {
-
+      tasks.push(createAddIntercomDataFn(leads[name], email));
-  function nextLead() {
-    if (leadIndex < leadEmails.length) {
-      return updateLead(leadEmails[leadIndex++]);
     }
-    return done(null, numLeadsCreated);
   }
-  function saveNewLead(email) {
+  async.parallel(tasks, (err, results) => {
-    const newLeadData = leads[email]; 
+    return done(err);
-    const name = newLeadData.name || email;
+  });
-    const postData = {
+}
-      display_name: newLeadData.trial_organization || newLeadData.coco_schoolName ||name,
+
-      name: newLeadData.trial_organization || newLeadData.coco_schoolName || name,
+function updateExistingLead(lead, existingLead, done) {
-      contacts: [{
+  // console.log('DEBUG: updateExistingLead', existingLead.id);
-        emails: [{email: email}],
+  const putData = lead.getLeadPutData(existingLead);
-        name: name
+  const options = {
-      }],
+    uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/${existingLead.id}/`,
-      custom: {
+    body: JSON.stringify(putData)
-        lastUpdated: new Date()
+  };
+  request.put(options, (error, response, body) => {
+    if (error) return done(error);
+    const result = JSON.parse(body);
+    if (result.errors || result['field-errors']) {
+      console.error(`Update existing lead PUT error for ${lead.name}`);
+      console.error(body);
+      // console.log(putData);
+      return done();
+    }
+
+    // Add contacts
+    const newContacts = lead.getContactsPostData(existingLead);
+    const tasks = []
+    for (const newContact of newContacts) {
+      newContact.lead_id = existingLead.id;
+      tasks.push(createAddContactFn(newContact));
+    }
+    async.parallel(tasks, (err, results) => {
+      if (err) return done(err);
+
+      // Add notes
+      const url = `https://${closeIoApiKey}:X@app.close.io/api/v1/activity/note/?lead_id=${existingLead.id}`;
+      request.get(url, (error, response, body) => {
+        if (error) return done(error);
+        const currentNotes = JSON.parse(body).data;
+        const newNotes = lead.getNotesPostData(currentNotes);
+        const tasks = []
+        for (const newNote of newNotes) {
+          tasks.push(createAddNoteFn(existingLead.id, newNote));
+        }
+        async.parallel(tasks, (err, results) => {
+          return done(err);
+        });
+      });
+    });
+  });
+}
+
+function saveNewLead(lead, done) {
+  // console.log('DEBUG: saveNewLead', lead.name);
+  const postData = lead.getLeadPostData();
+  const options = {
+    uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/`,
+    body: JSON.stringify(postData)
+  };
+  request.post(options, (error, response, body) => {
+    if (error) return done(error);
+    const existingLead = JSON.parse(body);
+    if (existingLead.errors || existingLead['field-errors']) {
+      console.error(`New lead POST error for ${lead.name}`);
+      console.error(body);
+      // console.error(JSON.stringify(postData, null, 2));
+      return done();
+    }
+
+    // Add notes
+    const newNotes = lead.getNotesPostData();
+    const tasks = []
+    for (const newNote of newNotes) {
+      tasks.push(createAddNoteFn(existingLead.id, newNote));
+    }
+    async.parallel(tasks, (err, results) => {
+      return done(err);
+    });
+  });
+}
+
+
+function createUpdateLeadFn(lead) {
+  return (done) => {
+    // console.log('DEBUG: updateLead', lead.name);
+    const url = `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/?query=name:${encodeURIComponent(lead.name)}`;
+    request.get(url, (error, response, body) => {
+      if (error) return done(error);
+      const data = JSON.parse(body);
+      if (data.total_results === 0) {
+        return saveNewLead(lead, done);
       }
-    };
+      if (data.total_results > 1) {
-    if (newLeadData.trialData && newLeadData.trial_phoneNumber) {
+        // console.error(`${data.total_results} leads found for ${lead.name}`);
-      postData.contacts[0].phones = [{phone: newLeadData.trial_phoneNumber}];
+        return done();
-    }
+      }
-    for (const key in newLeadData) {
+      return updateExistingLead(lead, data.data[0], done);
-      postData.custom[key] = newLeadData[key];
+    });
-    }
+  };
+}
+
+function createAddContactFn(postData) {
+  return (done) => {
+    // console.log('DEBUG: addContact', postData.lead_id);
     const options = {
-      uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/`,
+      uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/activity/Contact/`,
       body: JSON.stringify(postData)
     };
     request.post(options, (error, response, body) => {
       if (error) return done(error);
       const result = JSON.parse(body);
       if (result.errors || result['field-errors']) {
-        console.error(`New lead POST error for ${email}`);
+        console.error(`New Contact POST error for ${leadId}`);
         console.error(body);
       }
-      else {
+      return done();
-        numLeadsCreated++;
-      }
-      return nextLead();
     });
-  }
+  };
-  function updateLead(email) {
+}
-    const url = `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/?query=email_address:${encodeURIComponent(email)}`;
-    request.get(url, (error, response, body) => {
-      if (error) return done(error);
-      const data = JSON.parse(body);
-      if (data.total_results === 0) {
-        return saveNewLead(email);
-      }
-      return nextLead();
-    });
-  }
 
-  updateLead(leadEmails[leadIndex++]);
+function createAddNoteFn(leadId, newNote) {
+  return (done) => {
+    // console.log('DEBUG: addNote', leadId);
+    const notePostData = {
+      note: newNote,
+      lead_id: leadId
+    };
+    const options = {
+      uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/activity/note/`,
+      body: JSON.stringify(notePostData)
+    };
+    request.post(options, (error, response, body) => {
+      if (error) return done(error);
+      const result = JSON.parse(body);
+      if (result.errors || result['field-errors']) {
+        console.error(`New note POST error for ${leadId}`);
+        console.error(body);
+        // console.error(notePostData);
+      }
+      return done();
+    });
+  };
+}
+
+function updateLeads(leads, done) {
+  const tasks = []
+  for (const name in leads) {
+    if (leadsToSkip.indexOf(name) >= 0) continue;
+    tasks.push(createUpdateLeadFn(leads[name]));
+  }
+  async.parallel(tasks, (err, results) => {
+    return done(err);
+  });
 }
 
 function log(str) {
   console.log(new Date().toISOString() + " " + str);
 }
-