mirror of
https://github.com/codeninjasllc/discourse.git
synced 2024-11-27 09:36:19 -05:00
More entropy for foreign titles
* Treat strings with non-ASCII characters as having more entropy
This commit is contained in:
parent
5217602ec3
commit
bb77d2c38b
2 changed files with 12 additions and 2 deletions
|
@ -21,8 +21,10 @@ class TextSentinel
|
|||
end
|
||||
|
||||
# Entropy is a number of how many unique characters the string needs.
|
||||
# Non-ASCII characters are weighted heavier since they contain more "information"
|
||||
def entropy
|
||||
@entropy ||= @text.to_s.strip.split('').uniq.size
|
||||
chars = @text.to_s.strip.split('')
|
||||
@entropy ||= chars.pack('M*'*chars.size).gsub("\n",'').split('=').uniq.size
|
||||
end
|
||||
|
||||
def valid?
|
||||
|
|
|
@ -32,7 +32,15 @@ describe TextSentinel do
|
|||
end
|
||||
|
||||
it "Works on foreign characters" do
|
||||
TextSentinel.new("去年十社會警告").entropy.should == 7
|
||||
TextSentinel.new("去年十社會警告").entropy.should == 19
|
||||
end
|
||||
|
||||
it "generates enough entropy for short foreign strings" do
|
||||
TextSentinel.new("又一个测").entropy.should == 11
|
||||
end
|
||||
|
||||
it "handles repeated foreign characters" do
|
||||
TextSentinel.new("又一个测试话题" * 3).entropy.should == 18
|
||||
end
|
||||
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue