CHANGE: We now include the _escaped_fragment_ support by default, but

only if the crawler check fails. It is a fallback for non-google search
engines that support the Ajax crawling API.

app/controllers/application_controller.rb

@@ -39,8 +39,12 @@ class ApplicationController < ActionController::Base
 
   layout :set_layout
 
+  def has_escaped_fragment?
+    SiteSetting.enable_escaped_fragments? && params.key?("_escaped_fragment_")
+  end
+
   def set_layout
-    CrawlerDetection.crawler?(request.user_agent) ? 'crawler' : 'application'
+    has_escaped_fragment? || CrawlerDetection.crawler?(request.user_agent) ? 'crawler' : 'application'
   end
 
   rescue_from Exception do |exception|

app/controllers/topics_controller.rb

@@ -49,10 +49,6 @@ class TopicsController < ApplicationController
 
     redirect_to_correct_topic && return if slugs_do_not_match
 
-    # render workaround pseudo-static HTML page for old crawlers which ignores <noscript>
-    # (see http://meta.discourse.org/t/noscript-tag-and-some-search-engines/8078)
-    return render 'topics/plain', layout: false if (SiteSetting.enable_escaped_fragments && params.key?('_escaped_fragment_'))
-
     track_visit_to_topic
 
     if should_track_visit_to_topic?

app/helpers/application_helper.rb

@@ -86,10 +86,6 @@ module ApplicationHelper
       end
     end
 
-    # Add workaround tag for old crawlers which ignores <noscript>
-    # (see https://developers.google.com/webmasters/ajax-crawling/docs/specification)
-    result << tag('meta', name: "fragment", content: "!") if SiteSetting.enable_escaped_fragments
-
     result
   end
 

app/views/layouts/application.html.erb

@@ -7,6 +7,10 @@
     <meta content="<%= SiteSetting.site_description %>" name="description">
     <meta content="" name="author">
 
+    <%- if SiteSetting.enable_escaped_fragments? %>
+    <meta name="fragment" content="!">
+    <%- end %>
+
     <%= canonical_link_tag %>
 
     <link rel="icon" type="image/png" href="<%=SiteSetting.favicon_url%>">

config/locales/server.cs.yml

@@ -538,7 +538,7 @@ cs:
     must_approve_users: "Administrátoři musí schválit všechny uživatele, než získají přístup"
     ga_tracking_code: "Kód pro sledování přes 'Google analytics', např. UA-12345678-9; viz http://google.com/analytics"
     ga_domain_name: "Doménové jméno pro Google analytics, např. mysite.com; viz http://google.com/analytics"
-    enable_escaped_fragments: "Povolit alternativní řešení, které pomůže starým webovým robotům indexovat váš web. VAROVÁNÍ: povolte pouze v případě, že to opravdu potřebujete."
+    enable_escaped_fragments: "Povolit alternativní řešení, které pomůže starým webovým robotům indexovat váš web."
     enable_noscript_support: "Povolit podporu <noscript> tagu"
     top_menu: "Určuje, které položky se zobrazí v navigaci na hlavní stránce a v jakém pořadí. Příklad: latest|new|unread|starred|categories|top|read|posted"
     post_menu: "Určuje, které položky se zobrazí v menu u příspěvku a v jakém pořadí. Příklad: like|edit|flag|delete|share|bookmark|reply"

config/locales/server.da.yml

@@ -597,7 +597,7 @@ da:
     must_approve_users: "Admins skal godkende alle brugere før de får adgang"
     ga_tracking_code: "Google analytics sporingskode, f.eks.: UA-12345678-9; se http://google.com/analytics."
     ga_domain_name: "Google analytics domænenavn, f.eks.: example.com; se http://google.com/analytics."
-    enable_escaped_fragments: "Aktivér workaround som kan hjælpe ældre webcrawlers med at indeksere sitet. ADVARSEL: aktivér kun hvis du har behov for det."
+    enable_escaped_fragments: "Aktivér workaround som kan hjælpe ældre webcrawlers med at indeksere sitet."
     enable_noscript_support: "Aktivér standard webcrawler søgemaskineunderstøttelse vha. noscript-tagget"
     top_menu: "Angiv hvilke elementer der skal vises i hjemmesidenavigationen, og i hvilken rækkefølge. Eksempel: latest|new|unread|starred|categories|top|read|posted."
     post_menu: "Angiv hvilke elementer der skal vises i indlægsmenuen, og i hvilken rækkefølge. Eksempel: like|edit|flag|delete|share|bookmark|reply."

config/locales/server.de.yml

@@ -511,7 +511,7 @@ de:
     must_approve_users: "Administratoren müssen Nutzer freischalten, bevor sie Zugriff erlangen."
     ga_tracking_code: "Google Analytics Trackingcode, zum Beispiel: UA-12345678-9; siehe http://google.com/analytics"
     ga_domain_name: "Google Analytics Domänenname, zum Beispiel: mysite.com; siehe http://google.com/analytics"
-    enable_escaped_fragments: "Aktiviere Umgehungslösung um älteren Suchmaschinen-Webcrawler zu helfen die Seite zu indexieren. ACHTUNG: Nur aktivieren falls wirklich nötig."
+    enable_escaped_fragments: "Aktiviere Umgehungslösung um älteren Suchmaschinen-Webcrawler zu helfen die Seite zu indexieren."
     enable_noscript_support: "Aktiviere standard Suchmaschinen-Webcrawler Unterstützung durch den noscript Tag"
     top_menu: "Bestimme, welche Navigationselemente in welcher Reihenfolge auftauchen. Beispiel: latest|new|unread|starred|categories|top|read|posted"
     post_menu: "Bestimme, welche Funktionen in welcher Reihenfolge im Beitragsmenü auftauchen. Beispiel: like|edit|flag|delete|share|bookmark|reply"

config/locales/server.en.yml

@@ -629,7 +629,7 @@ en:
     ga_domain_name: "Google analytics (ga.js) domain name, eg: mysite.com; see http://google.com/analytics"
     ga_universal_tracking_code: "Google Universal Analytics (analytics.js) tracking code code, eg: UA-12345678-9; see http://google.com/analytics"
     ga_universal_domain_name: "Google Universal Analytics (analytics.js) domain name, eg: mysite.com; see http://google.com/analytics"
-    enable_escaped_fragments: "Enable workaround solution to help older search engine webcrawlers index your site. WARNING: enable only if you have to."
+    enable_escaped_fragments: "Fall back to Google's Ajax-Crawling API if a crawler is not detected."
     enable_noscript_support: "Enable standard webcrawler search engine support via the noscript tag"
     top_menu: "Determine which items appear in the homepage navigation, and in what order. Example latest|new|unread|starred|categories|top|read|posted"
     post_menu: "Determine which items appear on the post menu, and in what order. Example like|edit|flag|delete|share|bookmark|reply"

config/locales/server.fr.yml

@@ -550,7 +550,7 @@ fr:
     must_approve_users: "Les administrateurs du forum doivent approuver les utilisateurs avant qu'il y aient accès."
     ga_tracking_code: "Code de suivi de Google Analytics, par exemple : UA-12345678-9, voir: http://google.com/analytics"
     ga_domain_name: "Domaine suivi par Google Analytics, par exemple : monsite.com, voir: http://google.com/analytics"
-    enable_escaped_fragments: "Activer une solution de contournement afin d'aider d'ancien moteurs de recherche d'indexer votre site. ATTENTION : à activer seulement si vous le devez."
+    enable_escaped_fragments: "Activer une solution de contournement afin d'aider d'ancien moteurs de recherche d'indexer votre site."
     enable_noscript_support: "Activer le support des moteurs de recherches standards via le tag noscript"
     top_menu: "L'ordre des éléments dans le menu du haut. Exemple latest|new|unread|starred|categories|top|read|posted"
     post_menu: "L'ordre des éléments dans le menu de rédaction. Exemple like|edit|flag|delete|share|bookmark|reply"

config/locales/server.ja.yml

@@ -531,7 +531,7 @@ ja:
     must_approve_users: "管理者が全てのユーザの承認を行う"
     ga_tracking_code: "Google analytics のトラッキングコード。例: UA-12345678-9; 参考 http://google.com/analytics"
     ga_domain_name: "Google analytics のドメイン名。例: mysite.com; 参考 http://google.com/analytics"
-    enable_escaped_fragments: "古いサーチエンジンクローラがあなたのサイトのインデックスを構築するのをサポートするワークアラウンドを有効にする。注意: 必要な場合のみ有効にすることを推奨。"
+    enable_escaped_fragments: "古いサーチエンジンクローラがあなたのサイトのインデックスを構築するのをサポートするワークアラウンドを有効にする"
     enable_noscript_support: "noscript タグ経由でアクセスしてきた標準サーチエンジンクローラのサポートを有効にする"
     top_menu: "ホームページナビゲーションに表示する項目を指定。例 latest|new|unread|starred|categories|top|read|posted"
     post_menu: "ポストメニューに表示する項目を指定。例 like|edit|flag|delete|share|bookmark|reply"

config/locales/server.ko.yml

@@ -550,7 +550,7 @@ ko:
     must_approve_users: "Admins must approve all users before they gain access"
     ga_tracking_code: "Google analytics tracking code code, eg: UA-12345678-9; see http://google.com/analytics"
     ga_domain_name: "Google analytics domain name, eg: mysite.com; see http://google.com/analytics"
-    enable_escaped_fragments: "당신의 사이트의 오래된 정보를 검색하기 위한 웹크롤러 Workaround 솔루션을 활성. 주의: 꼭 필요할 때만 활성화"
+    enable_escaped_fragments: "당신의 사이트의 오래된 정보를 검색하기 위한 웹크롤러 Workaround 솔루션을 활성"
     enable_noscript_support: "당신의 사이트를 검색하기 위한 noscript 테그를 이용한 일반적인 웹 크롤러를 활성화"
     top_menu: "Determine which items appear in the homepage navigation, and in what order. Example latest|new|unread|starred|categories|top|read|posted"
     post_menu: "게시물 메뉴에 어떤 사항들이 무슨 순서로 올라올지 결정합니다. 예: 좋아요|수정|신고|삭제|공유|북마크|답글"

config/locales/server.nl.yml

@@ -600,7 +600,7 @@ nl:
     must_approve_users: "De eigenaars van een forum moeten de leden accepteren voordat zij toegang krijgen."
     ga_tracking_code: "Google analytics trackingcode, zie: http://google.com/analytics"
     ga_domain_name: "Google analytics domeinnaam, bijv. mijnsite.nl; zie http://google.com/analytics"
-    enable_escaped_fragments: "Schakel workaround in om oude crawlers je site te kunnen laten indexeren. WAARSCHUWING: gebruik dit alleen als je het echt nodig hebt."
+    enable_escaped_fragments: "Schakel workaround in om oude crawlers je site te kunnen laten indexeren."
     enable_noscript_support: "Ondersteun <noscript> tag"
     top_menu: "De volgorde en selectie van items in het hoofdmenu. Bijvoorbeeld latest|new|unread|starred|categories|top|read|posted"
     post_menu: "De volgorde en selectie van items in het berichtmenu. Bijvoorbeeld like|edit|flag|delete|share|bookmark|reply"

config/locales/server.pseudo.yml

@@ -708,7 +708,7 @@ pseudo:
     ga_domain_name: '[[ Ǧóóǧłé áɳáłýťíčš ďóɱáíɳ ɳáɱé, éǧ: ɱýšíťé.čóɱ; šéé ĥťťƿ://ǧóóǧłé.čóɱ/áɳáłýťíčš
       ]]'
     enable_escaped_fragments: '[[ Éɳáƀłé ŵóřǩářóůɳď šółůťíóɳ ťó ĥéłƿ ółďéř šéářčĥ
-      éɳǧíɳé ŵéƀčřáŵłéřš íɳďéх ýóůř šíťé. ŴÁŘЍÍЍǦ: éɳáƀłé óɳłý íƒ ýóů ĥáνé ťó. ]]'
+      éɳǧíɳé ŵéƀčřáŵłéřš íɳďéх ýóůř šíťé. ]]'
     enable_noscript_support: '[[ Éɳáƀłé šťáɳďářď ŵéƀčřáŵłéř šéářčĥ éɳǧíɳé šůƿƿóřť
       νíá ťĥé ɳóščříƿť ťáǧ ]]'
     top_menu: '[[ Ďéťéřɱíɳé ŵĥíčĥ íťéɱš áƿƿéář íɳ ťĥé ĥóɱéƿáǧé ɳáνíǧáťíóɳ, áɳď íɳ

config/locales/server.pt_BR.yml

@@ -542,7 +542,7 @@ pt_BR:
     must_approve_users: "Os proprietários do fórum tem que aprovar os usuários antes deles terem acesso."
     ga_tracking_code: "Código de rastreamento do Google Analytics, veja: http://google.com/analytics"
     ga_domain_name: "Nome de domínio no Google Analytics, exemplo: meusite.com; see http://google.com/analytics"
-    enable_escaped_fragments: "Ativa solução alternativa que ajuda os mecanismos de busca antigos a indexarem o seu site. ALERTA: apenas ative se você realmente precisa disto."
+    enable_escaped_fragments: "Ativa solução alternativa que ajuda os mecanismos de busca antigos a indexarem o seu site."
     enable_noscript_support: "Ativar suporte a tag <noscript>"
     top_menu: "A ordem dos items no menu de cima. Exemplo latest|new|unread|starred|categories|top|read|posted"
     post_menu: "A ordem dos items no menu da postagem."

config/locales/server.ru.yml

@@ -550,7 +550,7 @@ ru:
     must_approve_users: 'Администраторы должны одобрять учетные записи всех новых пользователей для того, чтобы они получили доступ'
     ga_tracking_code: 'Google analytics tracking code, например: UA-12345678-9; смотрите http://google.com/analytics'
     ga_domain_name: 'Доменное имя для Google analytics, например: mysite.com; смотрите http://google.com/analytics'
-    enable_escaped_fragments: 'Включить поддержку escaped_fragments для устаревших поисковых роботов (типа Яндекс). Внимание: включайте только при крайней необходимости.'
+    enable_escaped_fragments: 'Включить поддержку escaped_fragments для устаревших поисковых роботов (типа Яндекс).'
     enable_noscript_support: 'Включить поддержку тэга noscript для поисковых роботов'
     top_menu: 'Определите, как должны располагаться элементы в навигации на главной странице. Пример: latest|new|unread|starred|categories|top|read|posted'
     post_menu: 'Определите, какие элементы должны отображаться в меню у сообщения и в какой последовательности.  Пример: like|edit|flag|delete|share|bookmark|reply'

config/site_settings.yml

@@ -300,7 +300,7 @@ security:
   use_https: false
   enable_escaped_fragments:
     client: true
-    default: false
+    default: true
 
 seo:
   add_rel_nofollow_to_user_content: true

spec/controllers/application_controller_spec.rb

@@ -31,6 +31,67 @@ describe TopicsController do
     }.should change(IncomingLink, :count).by(1)
   end
 
+  describe "has_escaped_fragment?" do
+    render_views
+
+    context "when the SiteSetting is disabled" do
+      before do
+        SiteSetting.stubs(:enable_escaped_fragments?).returns(false)
+      end
+
+      it "uses the application layout even with an escaped fragment param" do
+        get :show, {'id' => topic.id, '_escaped_fragment_' => 'true'}
+        response.should render_template(layout: 'application')
+        assert_select "meta[name=fragment]", false, "it doesn't have the meta tag"
+      end
+    end
+
+    context "when the SiteSetting is enabled" do
+      before do
+        SiteSetting.stubs(:enable_escaped_fragments?).returns(true)
+      end
+
+      it "uses the application layout when there's no param" do
+        get :show, {'id' => topic.id}
+        response.should render_template(layout: 'application')
+        assert_select "meta[name=fragment]", true, "it has the meta tag"
+      end
+
+      it "uses the crawler layout when there's an _escaped_fragment_ param" do
+        get :show, {'id' => topic.id, '_escaped_fragment_' => 'true'}
+        response.should render_template(layout: 'crawler')
+        assert_select "meta[name=fragment]", false, "it doesn't have the meta tag"
+      end
+    end
+  end
+
+  describe "crawler" do
+    render_views
+
+    context "when not a crawler" do
+      before do
+        CrawlerDetection.expects(:crawler?).returns(false)
+      end
+      it "renders with the application layout" do
+        get :show, {'id' => topic.id}
+        response.should render_template(layout: 'application')
+        assert_select "meta[name=fragment]", true, "it has the meta tag"
+      end
+    end
+
+    context "when a crawler" do
+      before do
+        CrawlerDetection.expects(:crawler?).returns(true)
+      end
+      it "renders with the crawler layout" do
+        get :show, {'id' => topic.id}
+        response.should render_template(layout: 'crawler')
+        assert_select "meta[name=fragment]", false, "it doesn't have the meta tag"
+      end
+    end
+
+  end
+
   describe 'after inserting an incoming link' do
 
     it 'sets last link correctly' do