Update admin subs dashboard

Using invoices to aggregate totals which should provide a more accurate
growth rate.

app/templates/admin/analytics-subscriptions.jade

@@ -91,6 +91,7 @@ block content
             th Started
             th Cancelled
             th Net
+            th Ended
         tbody
           each sub in subs
             tr
@@ -99,3 +100,4 @@ block content
               td= sub.started
               td= sub.cancelled
               td= sub.started - sub.cancelled
+              td= sub.ended

app/views/admin/AnalyticsSubscriptionsView.coffee

@@ -87,16 +87,24 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
           subDayMap[cancelDay] ?= {}
           subDayMap[cancelDay]['cancel'] ?= 0
           subDayMap[cancelDay]['cancel']++
+        if endDay = sub?.end?.substring(0, 10)
+          subDayMap[endDay] ?= {}
+          subDayMap[endDay]['end'] ?= 0
+          subDayMap[endDay]['end']++
+      today = new Date().toISOString().substring(0, 10)
       for day of subDayMap
+        continue if day > today
         @subs.push
           day: day
           started: subDayMap[day]['start'] or 0
           cancelled: subDayMap[day]['cancel'] or 0
+          ended: subDayMap[day]['end'] or 0
 
       @subs.sort (a, b) -> a.day.localeCompare(b.day)
       startedLastMonth = 0
       for sub, i in @subs
         @total += sub.started
+        @total -= sub.ended
         @cancelled += sub.cancelled
         sub.total = @total
         startedLastMonth += sub.started if @subs.length - i < 31
@@ -229,7 +237,7 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
       lineColor: lineMetadata[startedSubsID].color
       strokeWidth: lineMetadata[startedSubsID].strokeWidth
       min: 0
-      max: d3.max(@subs, (d) -> d.started)
+      max: d3.max(@subs[-timeframeDays..], (d) -> d.started + 2)
 
     ## Total subs target
 
@@ -256,21 +264,29 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
       max: @targetSubCount
 
     ## Cancelled
+
+    # TODO: move this average cancelled stuff up the chain
     averageCancelled = 0
 
     # Build line data
     levelPoints = []
     cancelled = []
-    for sub, i in @subs
+    for sub, i in @subs[@subs.length - 30...]
-      if i >= @subs.length - 30
+      cancelled.push sub.cancelled
-        cancelled.push sub.cancelled
       levelPoints.push
-        x: i
+        x: @subs.length - 30 + i
         y: sub.cancelled
         day: sub.day
+        pointID: "#{cancelledSubsID}#{@subs.length - 30 + i}"
+        values: []
+    averageCancelled = cancelled.reduce((a, b) -> a + b) / cancelled.length
+    for sub, i in @subs[0...-30]
+      levelPoints.splice i, 0,
+        x: i
+        y: averageCancelled
+        day: sub.day
         pointID: "#{cancelledSubsID}#{i}"
         values: []
-    averageCancelled = cancelled.reduce((a, b) -> a + b) / cancelled.length
 
     # Ensure points for each day
     for day, i in days
@@ -293,7 +309,7 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
       lineColor: lineMetadata[cancelledSubsID].color
       strokeWidth: lineMetadata[cancelledSubsID].strokeWidth
       min: 0
-      max: d3.max(@subs, (d) -> d.started)
+      max: d3.max(@subs[-timeframeDays..], (d) -> d.started + 2)
 
     ## 7-Day Net Subs
 
@@ -338,8 +354,7 @@ module.exports = class AnalyticsSubscriptionsView extends RootView
       lineColor: lineMetadata[netSubsID].color
       strokeWidth: lineMetadata[netSubsID].strokeWidth
       min: 0
-      max: d3.max(@subs, (d) -> d.started)
+      max: d3.max(@subs[-timeframeDays..], (d) -> d.started + 2)
-
 
   updateAnalyticsGraphs: ->
     # Build d3 graphs

server/payments/subscription_handler.coffee

@@ -90,60 +90,83 @@ class SubscriptionHandler extends Handler
 
   getSubscriptions: (req, res) ->
     # Returns a list of active subscriptions
-    # TODO: does not handle customers with 11+ active subscriptions
     # TODO: does not track sponsored subs, only basic
     # TODO: does not return free subs
     # TODO: add tests
     # TODO: aggregate this data daily instead of providing it on demand
     # TODO: take date range as input
+    # TODO: are ended counts correct for today?  E.g. retries may complicate things.
 
     return @sendForbiddenError(res) unless req.user and req.user.isAdmin()
 
-    # @subs ?= []
     # return @sendSuccess(res, @subs) unless _.isEmpty(@subs)
-    @subs = []
+    @subMap = {}
 
-    customersProcessed = 0
+    console.log 'Fetching invoices...'
-    nextBatch = (starting_after, done) =>
+
+    processInvoices = (starting_after, done) =>
       options = limit: 100
       options.starting_after = starting_after if starting_after
-      stripe.customers.list options, (err, customers) =>
+      stripe.invoices.list options, (err, invoices) =>
         return done(err) if err
-        customersProcessed += customers.data.length
+        for invoice in invoices.data
-
+          continue unless invoice.paid
-        for customer in customers.data
+          continue unless invoice.subscription
-          continue unless customer?.subscriptions?.data?.length > 0
+          continue unless invoice.total > 0
-          for subscription in customer.subscriptions.data
+          continue unless invoice.lines?.data?[0]?.plan?.id is 'basic'
-            continue unless subscription.plan.id is 'basic'
+          subID = invoice.subscription
-
+          invoiceDate = new Date(invoice.date * 1000)
-            amount = subscription.plan.amount
+          if subID of @subMap
-            if subscription?.discount?.coupon?
+            @subMap[subID].first = invoiceDate
-              if subscription.discount.coupon.percent_off
+          else
-                amount = amount *  (100 - subscription.discount.coupon.percent_off) / 100;
+            @subMap[subID] =
-              else if subscription.discount.coupon.amount_off
+              first: invoiceDate
-                amount -= subscription.discount.coupon.amount_off
+              last: invoiceDate
-            else if customer.discount?.coupon?
+              customerID: invoice.customer
-              if customer.discount.coupon.percent_off
+        if invoices.has_more
-                amount = amount *  (100 - customer.discount.coupon.percent_off) / 100
+          console.log 'Fetching more invoices', Object.keys(@subMap).length
-              else if customer.discount.coupon.amount_off
+          return processInvoices(invoices.data[invoices.data.length - 1].id, done)
-                amount -= customer.discount.coupon.amount_off
-
-            continue unless amount > 0
-
-            sub = start: new Date(subscription.start * 1000)
-            if subscription.cancel_at_period_end
-              sub.cancel = new Date(subscription.canceled_at * 1000)
-              sub.end = new Date(subscription.current_period_end * 1000)
-            @subs.push(sub)
-
-        # Can't fetch all the test Stripe data
-        if customers.has_more and (config.isProduction or customersProcessed < 500)
-          return nextBatch(customers.data[customers.data.length - 1].id, done)
         else
           return done()
-    nextBatch null, (err) =>
+
+    processInvoices null, (err) =>
       return @sendDatabaseError(res, err) if err
-      @sendSuccess(res, @subs)
+
+      console.log 'Checking cancelled subscriptions...'
+
+      createCheckCancelledFn = (customerID, subscriptionID) =>
+        (done) =>
+          stripe.customers.retrieveSubscription customerID, subscriptionID, (err, subscription) =>
+            return done() if err
+            if subscription?.cancel_at_period_end
+              @subMap[subscriptionID].cancel = new Date(subscription.canceled_at * 1000)
+            done()
+
+      tasks = []
+      for subID of @subMap
+        expectedLastPayment = new Date(@subMap[subID].last)
+        expectedLastPayment.setUTCFullYear(new Date().getUTCFullYear())
+        expectedLastPayment.setUTCMonth(new Date().getUTCMonth() - 1)
+        expectedLastPayment.setUTCDate(new Date().getUTCDate() - 8) # In case last payment had some retries
+        if @subMap[subID].last > expectedLastPayment
+          tasks.push createCheckCancelledFn(@subMap[subID].customerID, subID)
+
+      async.parallel tasks, (err, results) =>
+        return @sendDatabaseError(res, err) if err
+        @subs = []
+        for subID of @subMap
+          sub =
+            start: @subMap[subID].first
+            subID: subID
+            customerID: @subMap[subID].customerID
+          sub.cancel = @subMap[subID].cancel if @subMap[subID].cancel
+          oneMonthAgo = new Date()
+          oneMonthAgo.setUTCMonth(oneMonthAgo.getUTCMonth() - 1)
+          if @subMap[subID].last < oneMonthAgo
+            sub.end = new Date(@subMap[subID].last)
+            sub.end.setUTCMonth(sub.end.getUTCMonth() + 1)
+          @subs.push sub
+        @sendSuccess(res, @subs)
 
   subscribeUser: (req, user, done) ->
     if (not req.user) or req.user.isAnonymous() or user.isAnonymous()