codeninjasllc/discourse
1
0
Fork 0
mirror of https://github.com/codeninjasllc/discourse.git synced 2025-02-17 04:01:29 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

FIX: Change crawl size to 10k. Youtube for example doesn't work with the

Browse source 
first 1k
This commit is contained in:
Robin Ward 2014-04-07 16:03:47 -04:00
parent b9799a886d
commit 1e3faddfe4
1 changed files with 14 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
app/jobs/regular/crawl_topic_link.rb
View file

@ -58,7 +58,7 @@ module Jobs
# Using exceptions for flow control is really bad, but there really seems to # Using exceptions for flow control is really bad, but there really seems to
# be no sane way to get a stream to stop reading in Excon (or Net::HTTP for # be no sane way to get a stream to stop reading in Excon (or Net::HTTP for
# that matter!) # that matter!)
raise ReadEnough.new if result.size > 1024 raise ReadEnough.new if result.size > 1024 * 10
end end
Excon.get(uri.to_s, response_block: streamer, read_timeout: 20, headers: CrawlTopicLink.request_headers(uri)) Excon.get(uri.to_s, response_block: streamer, read_timeout: 20, headers: CrawlTopicLink.request_headers(uri))
result result
@ -71,24 +71,24 @@ module Jobs
raise Discourse::InvalidParameters.new(:topic_link_id) unless args[:topic_link_id].present? raise Discourse::InvalidParameters.new(:topic_link_id) unless args[:topic_link_id].present?
begin begin
topic_link = TopicLink.where(id: args[:topic_link_id], internal: false, crawled_at: nil).first topic_link = TopicLink.where(id: args[:topic_link_id], internal: false, crawled_at: nil).first
return if topic_link.blank? return if topic_link.blank?
crawled = false crawled = false
result = CrawlTopicLink.fetch_beginning(topic_link.url) result = CrawlTopicLink.fetch_beginning(topic_link.url)
doc = Nokogiri::HTML(result) doc = Nokogiri::HTML(result)
if doc if doc
title = doc.at('title').try(:inner_text) title = doc.at('title').try(:inner_text)
if title.present?
title.gsub!(/\n/, ' ')
title.gsub!(/ +/, ' ')
title.strip!
if title.present? if title.present?
crawled = topic_link.update_attributes(title: title[0..255], crawled_at: Time.now) title.gsub!(/\n/, ' ')
title.gsub!(/ +/, ' ')
title.strip!
if title.present?
crawled = topic_link.update_attributes(title: title[0..255], crawled_at: Time.now)
end
end end
end end
end
rescue Exception rescue Exception
# If there was a connection error, do nothing # If there was a connection error, do nothing
ensure ensure