diff --git a/app/models/application_request.rb b/app/models/application_request.rb new file mode 100644 index 000000000..594f0b5a9 --- /dev/null +++ b/app/models/application_request.rb @@ -0,0 +1,86 @@ +class ApplicationRequest < ActiveRecord::Base + enum req_type: %i(anon logged_in crawler) + + cattr_accessor :autoflush + # auto flush if backlog is larger than this + self.autoflush = 100 + + def self.increment!(type, opts=nil) + key = redis_key(type) + val = $redis.incr(key).to_i + $redis.expire key, 3.days + + autoflush = (opts && opts[:autoflush]) || self.autoflush + if autoflush > 0 && val >= autoflush + write_cache! + end + end + + def self.write_cache!(date=nil) + if date.nil? + write_cache!(Time.now.utc) + write_cache!(Time.now.utc.yesterday) + return + end + + date = date.to_date + + # this may seem a bit fancy but in so it allows + # for concurrent calls without double counting + req_types.each do |req_type,_| + key = redis_key(req_type,date) + val = $redis.get(key).to_i + + next if val == 0 + + new_val = $redis.incrby(key, -val).to_i + + if new_val < 0 + # undo and flush next time + $redis.incrby(key, val) + next + end + + id = req_id(date,req_type) + + where(id: id).update_all(["count = count + ?", val]) + end + end + + def self.clear_cache!(date=nil) + if date.nil? + clear_cache!(Time.now.utc) + clear_cache!(Time.now.utc.yesterday) + return + end + + req_types.each do |req_type,_| + key = redis_key(req_type,date) + $redis.del key + end + end + + protected + + def self.req_id(date,req_type,retries=0) + + req_type_id = req_types[req_type] + + # a poor man's upsert + id = where(date: date, req_type: req_type_id).pluck(:id).first + id ||= create!(date: date, req_type: req_type_id, count: 0).id + + rescue # primary key violation + if retries == 0 + req_id(date,req_type,1) + else + raise + end + end + + def self.redis_key(req_type, time=Time.now.utc) + "app_req_#{req_type}#{time.strftime('%Y%m%d')}" + end + +end + diff --git a/db/migrate/20150203041207_add_application_requests.rb b/db/migrate/20150203041207_add_application_requests.rb new file mode 100644 index 000000000..32678edf8 --- /dev/null +++ b/db/migrate/20150203041207_add_application_requests.rb @@ -0,0 +1,11 @@ +class AddApplicationRequests < ActiveRecord::Migration + def change + create_table :application_requests do |t| + t.date :date, null: false + t.integer :req_type, null: false + t.integer :count, null: false, default: 0 + end + + add_index :application_requests, [:date, :req_type], unique: true + end +end diff --git a/spec/models/application_request_spec.rb b/spec/models/application_request_spec.rb new file mode 100644 index 000000000..67bc4247e --- /dev/null +++ b/spec/models/application_request_spec.rb @@ -0,0 +1,63 @@ +require 'spec_helper' + +describe ApplicationRequest do + + before do + ApplicationRequest.clear_cache! + end + + def inc(key,opts=nil) + ApplicationRequest.increment!(key,opts) + end + + it 'logs nothing for an unflushed increment' do + ApplicationRequest.increment!(:anon) + ApplicationRequest.count.should == 0 + end + + it 'can automatically flush' do + t1 = Time.now.utc.at_midnight + freeze_time(t1) + inc(:anon) + inc(:anon) + inc(:anon, autoflush: 3) + + ApplicationRequest.first.count.should == 3 + end + + it 'flushes yesterdays results' do + t1 = Time.now.utc.at_midnight + freeze_time(t1) + inc(:anon) + freeze_time(t1.tomorrow) + inc(:anon) + + ApplicationRequest.write_cache! + ApplicationRequest.count.should == 2 + end + + it 'clears cache correctly' do + # otherwise we have test pollution + inc(:anon) + ApplicationRequest.clear_cache! + ApplicationRequest.write_cache! + + ApplicationRequest.count.should == 0 + end + + it 'logs a few counts once flushed' do + time = Time.now.at_midnight + freeze_time(time) + + 3.times { inc(:anon) } + 2.times { inc(:logged_in) } + 4.times { inc(:crawler) } + + ApplicationRequest.write_cache! + + ApplicationRequest.anon.first.count.should == 3 + ApplicationRequest.logged_in.first.count.should == 2 + ApplicationRequest.crawler.first.count.should == 4 + + end +end