codeninjasllc/discourse
1
0
Fork 0
mirror of https://github.com/codeninjasllc/discourse.git synced 2024-11-23 23:58:31 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

REFACTOR: Rename GooglebotDetection to CrawlerDetection because we

Browse source 
will likely whitelist more crawlers in the future.
This commit is contained in:
Robin Ward 2014-02-20 16:07:02 -05:00
parent 7716d940a0
commit c4b5455c21
5 changed files with 34 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/controllers/application_controller.rb
View file

@ -4,7 +4,7 @@ require_dependency 'discourse'
require_dependency 'custom_renderer'
require_dependency 'archetype'
require_dependency 'rate_limiter'
require_dependency 'googlebot_detection'
require_dependency 'crawler_detection'
class ApplicationController < ActionController::Base
include CurrentUser
@ -40,7 +40,7 @@ class ApplicationController < ActionController::Base
layout :set_layout
def set_layout
GooglebotDetection.googlebot?(request.user_agent) ? 'googlebot' : 'application'
CrawlerDetection.crawler?(request.user_agent) ? 'crawler' : 'application'
end
rescue_from Exception do |exception|

0
app/views/layouts/googlebot.html.erb → app/views/layouts/crawler.html.erb
View file

4
lib/googlebot_detection.rb → lib/crawler_detection.rb
View file

@ -1,5 +1,5 @@
module GooglebotDetection
def self.googlebot?(user_agent)
module CrawlerDetection
def self.crawler?(user_agent)
!/Googlebot|Mediapartners|AdsBot/.match(user_agent).nil?
end
end

30
spec/components/crawler_detection_spec.rb Normal file
View file

@ -0,0 +1,30 @@
require 'spec_helper'
require_dependency 'crawler_detection'
describe CrawlerDetection do
describe "crawler?" do
it "returns true for crawler user agents" do
# https://support.google.com/webmasters/answer/1061943?hl=en
described_class.crawler?("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)").should == true
described_class.crawler?("Googlebot/2.1 (+http://www.google.com/bot.html)").should == true
described_class.crawler?("Googlebot-News").should == true
described_class.crawler?("Googlebot-Image/1.0").should == true
described_class.crawler?("Googlebot-Video/1.0").should == true
described_class.crawler?("(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)").should == true
described_class.crawler?("Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)").should == true
described_class.crawler?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)").should == true
described_class.crawler?("Mediapartners-Google").should == true
described_class.crawler?("AdsBot-Google (+http://www.google.com/adsbot.html)").should == true
end
it "returns false for non-crawler user agents" do
described_class.crawler?("Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36").should == false
described_class.crawler?("Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko").should == false
described_class.crawler?("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)").should == false
described_class.crawler?("Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25").should == false
described_class.crawler?("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0").should == false
described_class.crawler?("Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30").should == false
end
end
end

30
spec/components/googlebot_detection_spec.rb
View file

@ -1,30 +0,0 @@
require 'spec_helper'
require_dependency 'googlebot_detection'
describe GooglebotDetection do
describe "googlebot?" do
it "returns true for googlebot user agents" do
# https://support.google.com/webmasters/answer/1061943?hl=en
described_class.googlebot?("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)").should == true
described_class.googlebot?("Googlebot/2.1 (+http://www.google.com/bot.html)").should == true
described_class.googlebot?("Googlebot-News").should == true
described_class.googlebot?("Googlebot-Image/1.0").should == true
described_class.googlebot?("Googlebot-Video/1.0").should == true
described_class.googlebot?("(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)").should == true
described_class.googlebot?("Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)").should == true
described_class.googlebot?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)").should == true
described_class.googlebot?("Mediapartners-Google").should == true
described_class.googlebot?("AdsBot-Google (+http://www.google.com/adsbot.html)").should == true
end
it "returns false for non-googlebot user agents" do
described_class.googlebot?("Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36").should == false
described_class.googlebot?("Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko").should == false
described_class.googlebot?("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)").should == false
described_class.googlebot?("Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25").should == false
described_class.googlebot?("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0").should == false
described_class.googlebot?("Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30").should == false
end
end
end