From d298e2e065cb0b7aafad3e7193126f52c80f88da Mon Sep 17 00:00:00 2001 From: Neil Lalonde Date: Fri, 14 Feb 2014 17:10:08 -0500 Subject: [PATCH] Detect Googlebot from user agent and use a different layout that doesn't load javascript --- app/controllers/application_controller.rb | 7 ++++ app/views/layouts/googlebot.html.erb | 41 +++++++++++++++++++++ lib/googlebot_detection.rb | 5 +++ lib/mobile_detection.rb | 1 - spec/components/googlebot_detection_spec.rb | 30 +++++++++++++++ 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 app/views/layouts/googlebot.html.erb create mode 100644 lib/googlebot_detection.rb create mode 100644 spec/components/googlebot_detection_spec.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 5e25b5061..091cac085 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -4,6 +4,7 @@ require_dependency 'discourse' require_dependency 'custom_renderer' require_dependency 'archetype' require_dependency 'rate_limiter' +require_dependency 'googlebot_detection' class ApplicationController < ActionController::Base include CurrentUser @@ -36,6 +37,12 @@ class ApplicationController < ActionController::Base before_filter :check_xhr before_filter :redirect_to_login_if_required + layout :set_layout + + def set_layout + GooglebotDetection.googlebot?(request.user_agent) ? 'googlebot' : 'application' + end + rescue_from Exception do |exception| unless [ActiveRecord::RecordNotFound, ActionController::RoutingError, diff --git a/app/views/layouts/googlebot.html.erb b/app/views/layouts/googlebot.html.erb new file mode 100644 index 000000000..012c22fc4 --- /dev/null +++ b/app/views/layouts/googlebot.html.erb @@ -0,0 +1,41 @@ + + + + + <%= content_for?(:title) ? yield(:title) + ' - ' + SiteSetting.title : SiteSetting.title %> + + + + + <%= canonical_link_tag %> + + + + + <%= render :partial => "common/special_font_face" %> + <%= render :partial => "common/discourse_stylesheet" %> + <%= discourse_csrf_tags %> + <%= raw SiteContent.content_for(:head) %> + + + <%- unless customization_disabled? %> + <%= SiteCustomization.custom_header(session[:preview_style]) %> + <%- end %> +
+
+
+
+
+ +
+
+
+
+
+
+ + <%= yield %> + +
+ + diff --git a/lib/googlebot_detection.rb b/lib/googlebot_detection.rb new file mode 100644 index 000000000..a8164a4f7 --- /dev/null +++ b/lib/googlebot_detection.rb @@ -0,0 +1,5 @@ +module GooglebotDetection + def self.googlebot?(user_agent) + !/Googlebot|Mediapartners|AdsBot/.match(user_agent).nil? + end +end diff --git a/lib/mobile_detection.rb b/lib/mobile_detection.rb index b7501204e..0e00bb876 100644 --- a/lib/mobile_detection.rb +++ b/lib/mobile_detection.rb @@ -1,6 +1,5 @@ module MobileDetection def self.mobile_device?(user_agent) - # TODO: this is dumb. user agent matching is a doomed approach. a better solution is coming. user_agent =~ /Mobile|webOS|Nexus 7/ && !(user_agent =~ /iPad/) end diff --git a/spec/components/googlebot_detection_spec.rb b/spec/components/googlebot_detection_spec.rb new file mode 100644 index 000000000..59fb08c0d --- /dev/null +++ b/spec/components/googlebot_detection_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' +require_dependency 'googlebot_detection' + +describe GooglebotDetection do + describe "googlebot?" do + it "returns true for googlebot user agents" do + # https://support.google.com/webmasters/answer/1061943?hl=en + described_class.googlebot?("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)").should == true + described_class.googlebot?("Googlebot/2.1 (+http://www.google.com/bot.html)").should == true + described_class.googlebot?("Googlebot-News").should == true + described_class.googlebot?("Googlebot-Image/1.0").should == true + described_class.googlebot?("Googlebot-Video/1.0").should == true + described_class.googlebot?("(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)").should == true + described_class.googlebot?("Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)").should == true + described_class.googlebot?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)").should == true + described_class.googlebot?("Mediapartners-Google").should == true + described_class.googlebot?("AdsBot-Google (+http://www.google.com/adsbot.html)").should == true + end + + it "returns false for non-googlebot user agents" do + described_class.googlebot?("Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36").should == false + described_class.googlebot?("Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko").should == false + described_class.googlebot?("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)").should == false + described_class.googlebot?("Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25").should == false + described_class.googlebot?("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0").should == false + described_class.googlebot?("Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30").should == false + end + + end +end