From 86d1bc7f47abd2d9c3c2099774591fa6a7828db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Mon, 19 Sep 2016 13:43:15 +0200 Subject: [PATCH] new emoji:update rake task --- lib/tasks/emoji.rake | 142 ++++++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 50 deletions(-) diff --git a/lib/tasks/emoji.rake b/lib/tasks/emoji.rake index 729242f9b..894bf4640 100644 --- a/lib/tasks/emoji.rake +++ b/lib/tasks/emoji.rake @@ -1,63 +1,105 @@ +EMOJI_LIST_URL = "http://unicode.org/emoji/charts/full-emoji-list.html" +EMOJI_KEYWORDS_URL = "https://raw.githubusercontent.com/muan/emojilib/master/emojis.json" + desc "update emoji images" task "emoji:update" => :environment do - download_emojis_for("emoji_one", "https://raw.githubusercontent.com/Ranks/emojione/master/assets/png/%s.png", uppercase: true, leading_zeros: true) - download_emojis_for("twitter", "https://raw.githubusercontent.com/twitter/twemoji/gh-pages/72x72/%s.png", lowercase: true) - download_emojis_for("apple", "https://raw.githubusercontent.com/github/gemoji/master/images/emoji/unicode/%s.png", lowercase: true, leading_zeros: true) - # download_google_emojis("~/Desktop/images/%s.png") + emojis = {} + + puts "Loading local emoji database..." + db = JSON.parse(File.read("lib/emoji/db.json")) + db["emojis"].each do |e| + emojis[e["code"].tr("-", "_")] = { name: e["name"] } + end + aliases = db["aliases"].to_h + + puts "Enhancing emoji database with emojilib keywords..." + keywords = JSON.parse(open(EMOJI_KEYWORDS_URL).read) + keywords.keys.each do |k| + next unless char = keywords[k]["char"].presence + + code = char.codepoints + .map { |c| c.to_s(16).rjust(4, "0") } + .join("_") + .gsub(/_fe0f$/i, "") + + emojis[code] ||= { name: k } + end + + puts "Retrieving remote emoji list..." + list = open(EMOJI_LIST_URL).read + + puts "Parsing remote emoji list..." + doc = Nokogiri::HTML(list) + doc.css("tr").each do |row| + cells = row.css("td") + next if cells.size == 0 + + code = cells[1].at_css("a")["name"] + + next unless emojis[code] + + apple = cell_to_image(cells[4]) + google = cell_to_image(cells[5]) + twitter = cell_to_image(cells[6]) + one = cell_to_image(cells[7]) + windows = cell_to_image(cells[9]) + + if apple.blank? || google.blank? || twitter.blank? || one.blank? || windows.blank? + emojis.delete(code) + next + end + + emojis[code][:apple] = apple + emojis[code][:google] = google + emojis[code][:twitter] = twitter + emojis[code][:one] = one + emojis[code][:windows] = windows + end + + puts "Writing emojis..." + write_emojis(emojis, aliases, :apple, "apple") + write_emojis(emojis, aliases, :google, "google") + write_emojis(emojis, aliases, :twitter, "twitter") + write_emojis(emojis, aliases, :one, "emoji_one") + write_emojis(emojis, aliases, :windows, "win10") + + puts "Updating db.json..." + db = { + "emojis" => emojis.keys.map { |k| { "code" => k.tr("_", "-"), "name" => emojis[k][:name] } }, + "aliases" => aliases, + } + + File.write("lib/emoji/db.json", JSON.pretty_generate(db)) + + puts "Done!" end -def download_emojis_for(set, url_template, options={}) - puts "Downloading emojis for #{set}..." +def cell_to_image(cell) + return unless img = cell.at_css("img") + Base64.decode64(img["src"][/base64,(.+)$/, 1]) +end - path = "public/images/emoji/#{set}" - FileUtils.rm_rf(path) rescue nil - FileUtils.mkdir_p(path) rescue nil +def write_emojis(emojis, aliases, style, folder) + path = "public/images/emoji/#{folder}/" - uppercase = options[:uppercase] == true - lowercase = options[:lowercase] == true - leading_zeros = options[:leading_zeros] == true + FileUtils.rm_f Dir.glob("#{path}/*") - Emoji.all.each do |emoji| - codepoints = emoji["emoji"].codepoints.map { |c| c.to_s(16).rjust(leading_zeros ? 4 : 0, '0') } - filename = codepoints.join('-').sub(/-fe0f\b/, '') - filename = filename.downcase if lowercase - filename = filename.upcase if uppercase - puts "#{filename} -> #{emoji["emoji"]}" - url = url_template % filename - data = open(url).read rescue nil - next if data.nil? - FileUtils.cd(path) do - emoji["aliases"].each do |name| - File.open("#{name}.png", "wb") { |f| f << data } - ImageOptim.new.optimize_image!("#{name}.png") + puts folder + + emojis.values.each do |emoji| + write_emoji("#{path}/#{emoji[:name]}.png", emoji[style]) + if aliases[emoji[:name]] + aliases[emoji[:name]].each do |new_name| + write_emoji("#{path}/#{new_name}.png", emoji[style]) end end end + + puts end -# extracted from the NotoColorEmoji font -GOOGLE_EMOJIS ||= {35=>1, 48=>2, 49=>3, 50=>4, 51=>5, 52=>6, 53=>7, 54=>8, 55=>9, 56=>10, 57=>11, 169=>12, 174=>13, 8252=>14, 8265=>15, 8419=>16, 8482=>17, 8505=>18, 8596=>19, 8597=>20, 8598=>21, 8599=>22, 8600=>23, 8601=>24, 8617=>25, 8618=>26, 8986=>27, 8987=>28, 9193=>29, 9194=>30, 9195=>31, 9196=>32, 9200=>33, 9203=>34, 9410=>35, 9642=>36, 9643=>37, 9654=>38, 9664=>39, 9723=>40, 9724=>41, 9725=>42, 9726=>43, 9728=>44, 9729=>45, 9742=>46, 9745=>47, 9748=>48, 9749=>49, 9757=>50, 9786=>51, 9800=>52, 9801=>53, 9802=>54, 9803=>55, 9804=>56, 9805=>57, 9806=>58, 9807=>59, 9808=>60, 9809=>61, 9810=>62, 9811=>63, 9824=>64, 9827=>65, 9829=>66, 9830=>67, 9832=>68, 9851=>69, 9855=>70, 9875=>71, 9888=>72, 9889=>73, 9898=>74, 9899=>75, 9917=>76, 9918=>77, 9924=>78, 9925=>79, 9934=>80, 9940=>81, 9962=>82, 9970=>83, 9971=>84, 9973=>85, 9978=>86, 9981=>87, 9986=>88, 9989=>89, 9992=>90, 9993=>91, 9994=>92, 9995=>93, 9996=>94, 9999=>95, 10002=>96, 10004=>97, 10006=>98, 10024=>99, 10035=>100, 10036=>101, 10052=>102, 10055=>103, 10060=>104, 10062=>105, 10067=>106, 10068=>107, 10069=>108, 10071=>109, 10084=>110, 10133=>111, 10134=>112, 10135=>113, 10145=>114, 10160=>115, 10175=>116, 10548=>117, 10549=>118, 11013=>119, 11014=>120, 11015=>121, 11035=>122, 11036=>123, 11088=>124, 11093=>125, 12336=>126, 12349=>127, 12951=>128, 12953=>129, 126980=>130, 127183=>131, 127344=>132, 127345=>133, 127358=>134, 127359=>135, 127374=>136, 127377=>137, 127378=>138, 127379=>139, 127380=>140, 127381=>141, 127382=>142, 127383=>143, 127384=>144, 127385=>145, 127386=>146, 127462=>147, 127463=>148, 127464=>149, 127465=>150, 127466=>151, 127467=>152, 127468=>153, 127469=>154, 127470=>155, 127471=>156, 127472=>157, 127473=>158, 127474=>159, 127475=>160, 127476=>161, 127477=>162, 127478=>163, 127479=>164, 127480=>165, 127481=>166, 127482=>167, 127483=>168, 127484=>169, 127485=>170, 127486=>171, 127487=>172, 127489=>173, 127490=>174, 127514=>175, 127535=>176, 127538=>177, 127539=>178, 127540=>179, 127541=>180, 127542=>181, 127543=>182, 127544=>183, 127545=>184, 127546=>185, 127568=>186, 127569=>187, 127744=>188, 127745=>189, 127746=>190, 127747=>191, 127748=>192, 127749=>193, 127750=>194, 127751=>195, 127752=>196, 127753=>197, 127754=>198, 127755=>199, 127756=>200, 127757=>201, 127758=>202, 127759=>203, 127760=>204, 127761=>205, 127762=>206, 127763=>207, 127764=>208, 127765=>209, 127766=>210, 127767=>211, 127768=>212, 127769=>213, 127770=>214, 127771=>215, 127772=>216, 127773=>217, 127774=>218, 127775=>219, 127776=>220, 127792=>221, 127793=>222, 127794=>223, 127795=>224, 127796=>225, 127797=>226, 127799=>227, 127800=>228, 127801=>229, 127802=>230, 127803=>231, 127804=>232, 127805=>233, 127806=>234, 127807=>235, 127808=>236, 127809=>237, 127810=>238, 127811=>239, 127812=>240, 127813=>241, 127814=>242, 127815=>243, 127816=>244, 127817=>245, 127818=>246, 127819=>247, 127820=>248, 127821=>249, 127822=>250, 127823=>251, 127824=>252, 127825=>253, 127826=>254, 127827=>255, 127828=>256, 127829=>257, 127830=>258, 127831=>259, 127832=>260, 127833=>261, 127834=>262, 127835=>263, 127836=>264, 127837=>265, 127838=>266, 127839=>267, 127840=>268, 127841=>269, 127842=>270, 127843=>271, 127844=>272, 127845=>273, 127846=>274, 127847=>275, 127848=>276, 127849=>277, 127850=>278, 127851=>279, 127852=>280, 127853=>281, 127854=>282, 127855=>283, 127856=>284, 127857=>285, 127858=>286, 127859=>287, 127860=>288, 127861=>289, 127862=>290, 127863=>291, 127864=>292, 127865=>293, 127866=>294, 127867=>295, 127868=>296, 127872=>297, 127873=>298, 127874=>299, 127875=>300, 127876=>301, 127877=>302, 127878=>303, 127879=>304, 127880=>305, 127881=>306, 127882=>307, 127883=>308, 127884=>309, 127885=>310, 127886=>311, 127887=>312, 127888=>313, 127889=>314, 127890=>315, 127891=>316, 127904=>317, 127905=>318, 127906=>319, 127907=>320, 127908=>321, 127909=>322, 127910=>323, 127911=>324, 127912=>325, 127913=>326, 127914=>327, 127915=>328, 127916=>329, 127917=>330, 127918=>331, 127919=>332, 127920=>333, 127921=>334, 127922=>335, 127923=>336, 127924=>337, 127925=>338, 127926=>339, 127927=>340, 127928=>341, 127929=>342, 127930=>343, 127931=>344, 127932=>345, 127933=>346, 127934=>347, 127935=>348, 127936=>349, 127937=>350, 127938=>351, 127939=>352, 127940=>353, 127942=>354, 127943=>355, 127944=>356, 127945=>357, 127946=>358, 127968=>359, 127969=>360, 127970=>361, 127971=>362, 127972=>363, 127973=>364, 127974=>365, 127975=>366, 127976=>367, 127977=>368, 127978=>369, 127979=>370, 127980=>371, 127981=>372, 127982=>373, 127983=>374, 127984=>375, 128000=>376, 128001=>377, 128002=>378, 128003=>379, 128004=>380, 128005=>381, 128006=>382, 128007=>383, 128008=>384, 128009=>385, 128010=>386, 128011=>387, 128012=>388, 128013=>389, 128014=>390, 128015=>391, 128016=>392, 128017=>393, 128018=>394, 128019=>395, 128020=>396, 128021=>397, 128022=>398, 128023=>399, 128024=>400, 128025=>401, 128026=>402, 128027=>403, 128028=>404, 128029=>405, 128030=>406, 128031=>407, 128032=>408, 128033=>409, 128034=>410, 128035=>411, 128036=>412, 128037=>413, 128038=>414, 128039=>415, 128040=>416, 128041=>417, 128042=>418, 128043=>419, 128044=>420, 128045=>421, 128046=>422, 128047=>423, 128048=>424, 128049=>425, 128050=>426, 128051=>427, 128052=>428, 128053=>429, 128054=>430, 128055=>431, 128056=>432, 128057=>433, 128058=>434, 128059=>435, 128060=>436, 128061=>437, 128062=>438, 128064=>439, 128066=>440, 128067=>441, 128068=>442, 128069=>443, 128070=>444, 128071=>445, 128072=>446, 128073=>447, 128074=>448, 128075=>449, 128076=>450, 128077=>451, 128078=>452, 128079=>453, 128080=>454, 128081=>455, 128082=>456, 128083=>457, 128084=>458, 128085=>459, 128086=>460, 128087=>461, 128088=>462, 128089=>463, 128090=>464, 128091=>465, 128092=>466, 128093=>467, 128094=>468, 128095=>469, 128096=>470, 128097=>471, 128098=>472, 128099=>473, 128100=>474, 128101=>475, 128102=>476, 128103=>477, 128104=>478, 128105=>479, 128106=>480, 128107=>481, 128108=>482, 128109=>483, 128110=>484, 128111=>485, 128112=>486, 128113=>487, 128114=>488, 128115=>489, 128116=>490, 128117=>491, 128118=>492, 128119=>493, 128120=>494, 128121=>495, 128122=>496, 128123=>497, 128124=>498, 128125=>499, 128126=>500, 128127=>501, 128128=>502, 128129=>503, 128130=>504, 128131=>505, 128132=>506, 128133=>507, 128134=>508, 128135=>509, 128136=>510, 128137=>511, 128138=>512, 128139=>513, 128140=>514, 128141=>515, 128142=>516, 128143=>517, 128144=>518, 128145=>519, 128146=>520, 128147=>521, 128148=>522, 128149=>523, 128150=>524, 128151=>525, 128152=>526, 128153=>527, 128154=>528, 128155=>529, 128156=>530, 128157=>531, 128158=>532, 128159=>533, 128160=>534, 128161=>535, 128162=>536, 128163=>537, 128164=>538, 128165=>539, 128166=>540, 128167=>541, 128168=>542, 128169=>543, 128170=>544, 128171=>545, 128172=>546, 128173=>547, 128174=>548, 128175=>549, 128176=>550, 128177=>551, 128178=>552, 128179=>553, 128180=>554, 128181=>555, 128182=>556, 128183=>557, 128184=>558, 128185=>559, 128186=>560, 128187=>561, 128188=>562, 128189=>563, 128190=>564, 128191=>565, 128192=>566, 128193=>567, 128194=>568, 128195=>569, 128196=>570, 128197=>571, 128198=>572, 128199=>573, 128200=>574, 128201=>575, 128202=>576, 128203=>577, 128204=>578, 128205=>579, 128206=>580, 128207=>581, 128208=>582, 128209=>583, 128210=>584, 128211=>585, 128212=>586, 128213=>587, 128214=>588, 128215=>589, 128216=>590, 128217=>591, 128218=>592, 128219=>593, 128220=>594, 128221=>595, 128222=>596, 128223=>597, 128224=>598, 128225=>599, 128226=>600, 128227=>601, 128228=>602, 128229=>603, 128230=>604, 128231=>605, 128232=>606, 128233=>607, 128234=>608, 128235=>609, 128236=>610, 128237=>611, 128238=>612, 128239=>613, 128240=>614, 128241=>615, 128242=>616, 128243=>617, 128244=>618, 128245=>619, 128246=>620, 128247=>621, 128249=>622, 128250=>623, 128251=>624, 128252=>625, 128256=>626, 128257=>627, 128258=>628, 128259=>629, 128260=>630, 128261=>631, 128262=>632, 128263=>633, 128264=>634, 128265=>635, 128266=>636, 128267=>637, 128268=>638, 128269=>639, 128270=>640, 128271=>641, 128272=>642, 128273=>643, 128274=>644, 128275=>645, 128276=>646, 128277=>647, 128278=>648, 128279=>649, 128280=>650, 128281=>651, 128282=>652, 128283=>653, 128284=>654, 128285=>655, 128286=>656, 128287=>657, 128288=>658, 128289=>659, 128290=>660, 128291=>661, 128292=>662, 128293=>663, 128294=>664, 128295=>665, 128296=>666, 128297=>667, 128298=>668, 128299=>669, 128300=>670, 128301=>671, 128302=>672, 128303=>673, 128304=>674, 128305=>675, 128306=>676, 128307=>677, 128308=>678, 128309=>679, 128310=>680, 128311=>681, 128312=>682, 128313=>683, 128314=>684, 128315=>685, 128316=>686, 128317=>687, 128336=>688, 128337=>689, 128338=>690, 128339=>691, 128340=>692, 128341=>693, 128342=>694, 128343=>695, 128344=>696, 128345=>697, 128346=>698, 128347=>699, 128348=>700, 128349=>701, 128350=>702, 128351=>703, 128352=>704, 128353=>705, 128354=>706, 128355=>707, 128356=>708, 128357=>709, 128358=>710, 128359=>711, 128507=>712, 128508=>713, 128509=>714, 128510=>715, 128511=>716, 128512=>717, 128513=>718, 128514=>719, 128515=>720, 128516=>721, 128517=>722, 128518=>723, 128519=>724, 128520=>725, 128521=>726, 128522=>727, 128523=>728, 128524=>729, 128525=>730, 128526=>731, 128527=>732, 128528=>733, 128529=>734, 128530=>735, 128531=>736, 128532=>737, 128533=>738, 128534=>739, 128535=>740, 128536=>741, 128537=>742, 128538=>743, 128539=>744, 128540=>745, 128541=>746, 128542=>747, 128543=>748, 128544=>749, 128545=>750, 128546=>751, 128547=>752, 128548=>753, 128549=>754, 128550=>755, 128551=>756, 128552=>757, 128553=>758, 128554=>759, 128555=>760, 128556=>761, 128557=>762, 128558=>763, 128559=>764, 128560=>765, 128561=>766, 128562=>767, 128563=>768, 128564=>769, 128565=>770, 128566=>771, 128567=>772, 128568=>773, 128569=>774, 128570=>775, 128571=>776, 128572=>777, 128573=>778, 128574=>779, 128575=>780, 128576=>781, 128581=>782, 128582=>783, 128583=>784, 128584=>785, 128585=>786, 128586=>787, 128587=>788, 128588=>789, 128589=>790, 128590=>791, 128591=>792, 128640=>793, 128641=>794, 128642=>795, 128643=>796, 128644=>797, 128645=>798, 128646=>799, 128647=>800, 128648=>801, 128649=>802, 128650=>803, 128651=>804, 128652=>805, 128653=>806, 128654=>807, 128655=>808, 128656=>809, 128657=>810, 128658=>811, 128659=>812, 128660=>813, 128661=>814, 128662=>815, 128663=>816, 128664=>817, 128665=>818, 128666=>819, 128667=>820, 128668=>821, 128669=>822, 128670=>823, 128671=>824, 128672=>825, 128673=>826, 128674=>827, 128675=>828, 128676=>829, 128677=>830, 128678=>831, 128679=>832, 128680=>833, 128681=>834, 128682=>835, 128683=>836, 128684=>837, 128685=>838, 128686=>839, 128687=>840, 128688=>841, 128689=>842, 128690=>843, 128691=>844, 128692=>845, 128693=>846, 128694=>847, 128695=>848, 128696=>849, 128697=>850, 128698=>851, 128699=>852, 128700=>853, 128701=>854, 128702=>855, 128703=>856, 128704=>857, 128705=>858, 128706=>859, 128707=>860, 128708=>861, 128709=>862, 1041637=>978, 1041638=>1088, 1041639=>943, 1041640=>926, 1041641=>974, 1041642=>945, 1041643=>936, 1041644=>1050, 1041645=>917, 1041646=>986, 1042476=>863, 1042478=>865, 1042479=>866, 1042480=>867, 1042481=>868, 1042482=>869, 1042483=>870, 1042484=>871, 1042485=>872, 1042486=>873, 1042487=>864}.freeze - -def download_google_emojis(url_template) - puts "Downloading emojis for google..." - - path = "public/images/google" - FileUtils.rm_rf(path) rescue nil - FileUtils.mkdir_p(path) rescue nil - - Emoji.all.each do |emoji| - codepoint = emoji["emoji"].codepoints.first - filename = GOOGLE_EMOJIS[codepoint] - next if filename.nil? - puts "#{filename} -> #{emoji["emoji"]}" - url = url_template % filename - data = File.open(url, "rb").read - next if data.nil? - FileUtils.cd(path) do - emoji["aliases"].each do |name| - File.open("#{name}.png", "wb") { |f| f << data } - ImageOptim.new.optimize_image!("#{name}.png") - end - end - end +def write_emoji(path, emoji) + open(path, "wb") { |f| f << emoji } + `pngout #{path}` + putc "." end