discourse/app/controllers/robots_txt_controller.rb

class RobotsTxtController < ApplicationController
  layout false
  skip_before_filter :check_xhr

  def index
    path = if SiteSetting.allow_index_in_robots_txt && !SiteSetting.restrict_access
      :index
    else
      :no_index
    end
    
    render path, content_type: 'text/plain'
  end
end