Replace emoji from a LUT

.gitignore

@@ -1,9 +1,2 @@
-/bin/*
-!/bin/gemoji
-.bundle
 .ruby-version
-Gemfile.lock
-db/emoji-test.txt
-db/ucd.nounihan.grouped.xml
-images/unicode/*.png
-vendor/
+db/NamesList.txt

.travis.yml

@@ -1,9 +1,7 @@
-sudo: false
 script: script/test
 rvm:
   - 1.9.3
-  - '2.1'
-  - '2.2'
-  - 2.3.0
+  - 2.1.2
+  - 2.2
 notifications:
   email: false

CONTRIBUTING.md

@@ -1,6 +1,3 @@
-This project adheres to the [Open Code of Conduct][code-of-conduct]. By participating, you are expected to uphold this code.
-[code-of-conduct]: http://todogroup.org/opencodeofconduct/#gemoji/opensource@github.com
-
 Our emoji set is based off Apple's emoji character palette, plus some custom
 emoji such as :octocat: :shipit: :metal:.
 

Gemfile

@@ -1,6 +1,7 @@
 source "https://rubygems.org"
 
-gem "rake", "~> 10.3.2"
-gem "minitest", "~> 5.3.5"
+gem "ttfunk", "~> 1.4.0"
+gem "rake"
+gem "minitest"
 
 gemspec

Gemfile.lock

@@ -0,0 +1,20 @@
+PATH
+  remote: .
+  specs:
+    gemoji (2.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    minitest (5.3.5)
+    rake (10.3.2)
+    ttfunk (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  gemoji!
+  minitest
+  rake
+  ttfunk (~> 1.4.0)

LICENSE

@@ -10,6 +10,9 @@ Copyright (c) 2013 id Software. All rights reserved.
 trollface
 Copyright (c) 2013 whynne@deviantart. All rights reserved.
 
+All other images
+Copyright (c) 2013 Apple Inc. All rights reserved.
+
 Source code:
 
 Copyright (c) 2013 GitHub, Inc.

README.md

@@ -1,8 +1,7 @@
 gemoji
 ======
 
-This library contains character information about native emoji, as well as image
-files for a few custom emoji.
+Emoji images and names. See the LICENSE for copyright information.
 
 
 Installation
@@ -14,19 +13,36 @@ Add `gemoji` to your Gemfile.
 gem 'gemoji'
 ```
 
-### Extract images
+**Sync images**
 
-To obtain image files as fallbacks for browsers and OS's that don't support
-emoji, run the `gemoji extract` command **on macOS Sierra or later**:
+Images can be copied to your public directory with `rake emoji` in your app. This is the recommended approach since the images will be available at a consistent location. This works best with cached formatted user content generated by tools like [html-pipeline](https://github.com/jch/html-pipeline).
 
-``` sh
-bundle exec gemoji extract public/images/emoji
+``` ruby
+# Rakefile
+load 'tasks/emoji.rake'
 ```
 
-This will extract images into filenames such as:
+```
+$ rake emoji
+```
 
-* `public/images/emoji/octocat.png`
-* `public/images/emoji/unicode/1f9c0.png` (the `:cheese:` emoji)
+**Assets Precompiling**
+
+If you must, you can manually add all the images to your asset load path.
+
+``` ruby
+# config/application.rb
+config.assets.paths << Emoji.images_path
+```
+
+Then have them compiled to public on deploy.
+
+``` ruby
+# config/application.rb
+config.assets.precompile << "emoji/**/*.png"
+```
+
+**WARNING** Since there are a ton of images, just adding the path may slow down other lookups if you aren't using it. Compiling all the emojis on deploy will add overhead to your deploy if even the images haven't changed. Theres just so many more superfluous files to iterate over. Also, the urls will be fingerprinted which may not be ideal for referencing from cached content.
 
 
 Example Rails Helper
@@ -41,7 +57,7 @@ module EmojiHelper
   def emojify(content)
     h(content).to_str.gsub(/:([\w+-]+):/) do |match|
       if emoji = Emoji.find_by_alias($1)
-        %(<img alt="#$1" src="#{image_path("emoji/#{emoji.image_filename}")}" style="vertical-align:middle" width="20" height="20" />)
+        %(<img alt="#$1" src="#{asset_path("emoji/#{emoji.image_filename}")}" style="vertical-align:middle" width="20" height="20" />)
       else
         match
       end
@@ -91,14 +107,6 @@ emoji.image_filename #=> "music.png"
 As you create new emoji, you must ensure that you also create and put the images
 they reference by their `image_filename` to your assets directory.
 
-You can customize `image_filename` with:
-
-```ruby
-emoji = Emoji.create("music") do |char|
-  char.image_filename = "subdirectory/my_emoji.gif"
-end
-```
-
 For existing emojis, you can edit the list of aliases or add new tags in an edit block:
 
 ```ruby

Rakefile

@@ -1,4 +1,5 @@
 require 'rake/testtask'
+require 'rake/extensiontask'
 
 task :default => :test
 
@@ -9,11 +10,7 @@ end
 
 namespace :db do
   desc %(Generate Emoji data files needed for development)
-  task :generate => [
-    'db/Category-Emoji.json',
-    'db/ucd.nounihan.grouped.xml',
-    'db/emoji-test.txt',
-  ]
+  task :generate => ['db/Category-Emoji.json', 'db/NamesList.txt']
 
   desc %(Dump a list of supported Emoji with Unicode descriptions and aliases)
   task :dump => :generate do
@@ -21,25 +18,35 @@ namespace :db do
   end
 end
 
+emoji_plist = '/System/Library/Input Methods/CharacterPalette.app/Contents/Resources/Category-Emoji.plist'
+nameslist_url = 'http://www.unicode.org/Public/6.3.0/ucd/NamesList.txt'
+
 task 'db/Category-Emoji.json' do |t|
-  system 'plutil', '-convert', 'json', '-r',
-    '/System/Library/Input Methods/CharacterPalette.app/Contents/Resources/Category-Emoji.plist',
-    '-o', t.name
+  system "plutil -convert json -r '#{emoji_plist}' -o '#{t.name}'"
 end
 
-file 'db/ucd.nounihan.grouped.xml' do
-  Dir.chdir('db') do
-    system 'curl', '-fsSLO', 'http://www.unicode.org/Public/9.0.0/ucdxml/ucd.nounihan.grouped.zip'
-    system 'unzip', '-q', 'ucd.nounihan.grouped.zip'
-    rm 'ucd.nounihan.grouped.zip'
+file 'db/NamesList.txt' do |t|
+  system "curl -fsSL '#{nameslist_url}' -o '#{t.name}'"
+end
+
+namespace :images do
+  desc %(Extract PNG images from Apple's "Apple Color Emoji.ttf" font)
+  task :extract do
+    require 'emoji/extractor'
+    gem_dir = File.dirname(File.realpath(__FILE__))
+    Emoji::Extractor.new(64, "#{gem_dir}/images/emoji/unicode").extract!
   end
 end
 
-file 'db/emoji-test.txt' do |t|
-  system 'curl', '-fsSL', 'http://unicode.org/Public/emoji/4.0/emoji-test.txt', '-o', t.name
+namespace :c do
+  task :headers do
+    require 'emoji/tables'
+    gem_dir = File.dirname(File.realpath(__FILE__))
+
+    File.open(File.join(gem_dir, "ext/gemoji/emoji.h"), "w") do |file|
+      file.puts(Emoji::Tables.generate_length_tables)
+    end
+  end
 end
 
-directory 'images/unicode' do
-  require 'emoji/extractor'
-  Emoji::Extractor.new(64, Emoji.images_path).extract!
-end
+Rake::ExtensionTask.new('gemoji')

bin/gemoji

@@ -1,5 +0,0 @@
-#!/usr/bin/env ruby
-require 'emoji/cli'
-
-exit_code = Emoji::CLI.dispatch(ARGV)
-exit exit_code

db/Category-Emoji.json

@@ -4,64 +4,40 @@
       "CVDataTitle" : "EmojiCategory-People",
       "CVCategoryImage" : "Emoji-HumanImage",
       "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "😀,😃,😄,😁,😆,😅,😂,🤣,☺️,😊,😇,🙂,🙃,😉,😌,😍,😘,😗,😙,😚,😋,😜,😝,😛,🤑,🤗,🤓,😎,🤡,🤠,😏,😒,😞,😔,😟,😕,🙁,☹️,😣,😖,😫,😩,😤,😠,😡,😶,😐,😑,😯,😦,😧,😮,😲,😵,😳,😱,😨,😰,😢,😥,🤤,😭,😓,😪,😴,🙄,🤔,🤥,😬,🤐,🤢,🤧,😷,🤒,🤕,😈,👿,👹,👺,💩,👻,💀,☠️,👽,👾,🤖,🎃,😺,😸,😹,😻,😼,😽,🙀,😿,😾,👐,🙌,👏,🙏,🤝,👍,👎,👊,✊,🤛,🤜,🤞,✌️,🤘,👌,👈,👉,👆,👇,☝️,✋,🤚,🖐,🖖,👋,🤙,💪,🖕,✍️,🤳,💅,💍,💄,💋,👄,👅,👂,👃,👣,👁,👀,🗣,👤,👥,👶,👦,👧,👨,👩,👱‍♀️,👱,👴,👵,👲,👳‍♀️,👳,👮‍♀️,👮,👷‍♀️,👷,💂‍♀️,💂,🕵️‍♀️,🕵️,👩‍⚕️,👨‍⚕️,👩‍🌾,👨‍🌾,👩‍🍳,👨‍🍳,👩‍🎓,👨‍🎓,👩‍🎤,👨‍🎤,👩‍🏫,👨‍🏫,👩‍🏭,👨‍🏭,👩‍💻,👨‍💻,👩‍💼,👨‍💼,👩‍🔧,👨‍🔧,👩‍🔬,👨‍🔬,👩‍🎨,👨‍🎨,👩‍🚒,👨‍🚒,👩‍✈️,👨‍✈️,👩‍🚀,👨‍🚀,👩‍⚖️,👨‍⚖️,🤶,🎅,👸,🤴,👰,🤵,👼,🤰,🙇‍♀️,🙇,💁,💁‍♂️,🙅,🙅‍♂️,🙆,🙆‍♂️,🙋,🙋‍♂️,🤦‍♀️,🤦‍♂️,🤷‍♀️,🤷‍♂️,🙎,🙎‍♂️,🙍,🙍‍♂️,💇,💇‍♂️,💆,💆‍♂️,🕴,💃,🕺,👯,👯‍♂️,🚶‍♀️,🚶,🏃‍♀️,🏃,👫,👭,👬,💑,👩‍❤️‍👩,👨‍❤️‍👨,💏,👩‍❤️‍💋‍👩,👨‍❤️‍💋‍👨,👪,👨‍👩‍👧,👨‍👩‍👧‍👦,👨‍👩‍👦‍👦,👨‍👩‍👧‍👧,👩‍👩‍👦,👩‍👩‍👧,👩‍👩‍👧‍👦,👩‍👩‍👦‍👦,👩‍👩‍👧‍👧,👨‍👨‍👦,👨‍👨‍👧,👨‍👨‍👧‍👦,👨‍👨‍👦‍👦,👨‍👨‍👧‍👧,👩‍👦,👩‍👧,👩‍👧‍👦,👩‍👦‍👦,👩‍👧‍👧,👨‍👦,👨‍👧,👨‍👧‍👦,👨‍👦‍👦,👨‍👧‍👧,👚,👕,👖,👔,👗,👙,👘,👠,👡,👢,👞,👟,👒,🎩,🎓,👑,⛑,🎒,👝,👛,👜,💼,👓,🕶,🌂,☂️"
+        "CVSkipNullGlyphs" : true,
+        "Data" : "😄,😃,😀,😊,☺️,😉,😍,😘,😚,😗,😙,😜,😝,😛,😳,😁,😔,😌,😒,😞,😣,😢,😂,😭,😪,😥,😰,😅,😓,😩,😫,😨,😱,😠,😡,😤,😖,😆,😋,😷,😎,😴,😵,😲,😟,😦,😧,😈,👿,😮,😬,😐,😕,😯,😶,😇,😏,😑,👲,👳,👮,👷,💂,👶,👦,👧,👨,👩,👴,👵,👱,👼,👸,😺,😸,😻,😽,😼,🙀,😿,😹,😾,👹,👺,🙈,🙉,🙊,💀,👽,💩,🔥,✨,🌟,💫,💥,💢,💦,💧,💤,💨,👂,👀,👃,👅,👄,👍,👎,👌,👊,✊,✌️,👋,✋,👐,👆,👇,👉,👈,🙌,🙏,☝️,👏,💪,🚶,🏃,💃,👫,👪,👬,👭,💏,💑,👯,🙆,🙅,💁,🙋,💆,💇,💅,👰,🙎,🙍,🙇,🎩,👑,👒,👟,👞,👡,👠,👢,👕,👔,👚,👗,🎽,👖,👘,👙,💼,👜,👝,👛,👓,🎀,🌂,💄,💛,💙,💜,💚,❤️,💔,💗,💓,💕,💖,💞,💘,💌,💋,💍,💎,👤,👥,💬,👣,💭"
       }
     },
     {
       "CVDataTitle" : "EmojiCategory-Nature",
       "CVCategoryImage" : "Emoji-NatureImage",
       "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "🐶,🐱,🐭,🐹,🐰,🦊,🐻,🐼,🐨,🐯,🦁,🐮,🐷,🐽,🐸,🐵,🙈,🙉,🙊,🐒,🐔,🐧,🐦,🐤,🐣,🐥,🦆,🦅,🦉,🦇,🐺,🐗,🐴,🦄,🐝,🐛,🦋,🐌,🐚,🐞,🐜,🕷,🕸,🐢,🐍,🦎,🦂,🦀,🦑,🐙,🦐,🐠,🐟,🐡,🐬,🦈,🐳,🐋,🐊,🐆,🐅,🐃,🐂,🐄,🦌,🐪,🐫,🐘,🦏,🦍,🐎,🐖,🐐,🐏,🐑,🐕,🐩,🐈,🐓,🦃,🕊,🐇,🐁,🐀,🐿,🐾,🐉,🐲,🌵,🎄,🌲,🌳,🌴,🌱,🌿,☘️,🍀,🎍,🎋,🍃,🍂,🍁,🍄,🌾,💐,🌷,🌹,🥀,🌻,🌼,🌸,🌺,🌎,🌍,🌏,🌕,🌖,🌗,🌘,🌑,🌒,🌓,🌔,🌚,🌝,🌞,🌛,🌜,🌙,💫,⭐️,🌟,✨,⚡️,🔥,💥,☄️,☀️,🌤,⛅️,🌥,🌦,🌈,☁️,🌧,⛈,🌩,🌨,☃️,⛄️,❄️,🌬,💨,🌪,🌫,🌊,💧,💦,☔️"
-      }
-    },
-    {
-      "CVDataTitle" : "EmojiCategory-Foods",
-      "CVCategoryImage" : "Emoji-FoodsImage",
-      "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "🍏,🍎,🍐,🍊,🍋,🍌,🍉,🍇,🍓,🍈,🍒,🍑,🍍,🥝,🥑,🍅,🍆,🥒,🥕,🌽,🌶,🥔,🍠,🌰,🥜,🍯,🥐,🍞,🥖,🧀,🥚,🍳,🥓,🥞,🍤,🍗,🍖,🍕,🌭,🍔,🍟,🥙,🌮,🌯,🥗,🥘,🍝,🍜,🍲,🍥,🍣,🍱,🍛,🍚,🍙,🍘,🍢,🍡,🍧,🍨,🍦,🍰,🎂,🍮,🍭,🍬,🍫,🍿,🍩,🍪,🥛,🍼,☕️,🍵,🍶,🍺,🍻,🥂,🍷,🥃,🍸,🍹,🍾,🥄,🍴,🍽"
-      }
-    },
-    {
-      "CVDataTitle" : "EmojiCategory-Activity",
-      "CVCategoryImage" : "Emoji-ActivityImage",
-      "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "⚽️,🏀,🏈,⚾️,🎾,🏐,🏉,🎱,🏓,🏸,🥅,🏒,🏑,🏏,⛳️,🏹,🎣,🥊,🥋,⛸,🎿,⛷,🏂,🏋️‍♀️,🏋️,🤺,🤼‍♀️,🤼‍♂️,🤸‍♀️,🤸‍♂️,⛹️‍♀️,⛹️,🤾‍♀️,🤾‍♂️,🏌️‍♀️,🏌️,🏄‍♀️,🏄,🏊‍♀️,🏊,🤽‍♀️,🤽‍♂️,🚣‍♀️,🚣,🏇,🚴‍♀️,🚴,🚵‍♀️,🚵,🎽,🏅,🎖,🥇,🥈,🥉,🏆,🏵,🎗,🎫,🎟,🎪,🤹‍♀️,🤹‍♂️,🎭,🎨,🎬,🎤,🎧,🎼,🎹,🥁,🎷,🎺,🎸,🎻,🎲,🎯,🎳,🎮,🎰"
-      }
-    },
-    {
-      "CVDataTitle" : "EmojiCategory-Places",
-      "CVCategoryImage" : "Emoji-PlacesImage",
-      "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "🚗,🚕,🚙,🚌,🚎,🏎,🚓,🚑,🚒,🚐,🚚,🚛,🚜,🛴,🚲,🛵,🏍,🚨,🚔,🚍,🚘,🚖,🚡,🚠,🚟,🚃,🚋,🚞,🚝,🚄,🚅,🚈,🚂,🚆,🚇,🚊,🚉,🚁,🛩,✈️,🛫,🛬,🚀,🛰,💺,🛶,⛵️,🛥,🚤,🛳,⛴,🚢,⚓️,🚧,⛽️,🚏,🚦,🚥,🗺,🗿,🗽,⛲️,🗼,🏰,🏯,🏟,🎡,🎢,🎠,⛱,🏖,🏝,⛰,🏔,🗻,🌋,🏜,🏕,⛺️,🛤,🛣,🏗,🏭,🏠,🏡,🏘,🏚,🏢,🏬,🏣,🏤,🏥,🏦,🏨,🏪,🏫,🏩,💒,🏛,⛪️,🕌,🕍,🕋,⛩,🗾,🎑,🏞,🌅,🌄,🌠,🎇,🎆,🌇,🌆,🏙,🌃,🌌,🌉,🌁"
+        "CVSkipNullGlyphs" : true,
+        "Data" : "🐶,🐺,🐱,🐭,🐹,🐰,🐸,🐯,🐨,🐻,🐷,🐽,🐮,🐗,🐵,🐒,🐴,🐑,🐘,🐼,🐧,🐦,🐤,🐥,🐣,🐔,🐍,🐢,🐛,🐝,🐜,🐞,🐌,🐙,🐚,🐠,🐟,🐬,🐳,🐋,🐄,🐏,🐀,🐃,🐅,🐇,🐉,🐎,🐐,🐓,🐕,🐖,🐁,🐂,🐲,🐡,🐊,🐫,🐪,🐆,🐈,🐩,🐾,💐,🌸,🌷,🍀,🌹,🌻,🌺,🍁,🍃,🍂,🌿,🌾,🍄,🌵,🌴,🌲,🌳,🌰,🌱,🌼,🌐,🌞,🌝,🌚,🌑,🌒,🌓,🌔,🌕,🌖,🌗,🌘,🌜,🌛,🌙,🌍,🌎,🌏,🌋,🌌,🌠,⭐️,☀️,⛅️,☁️,⚡️,☔️,❄️,⛄️,🌀,🌁,🌈,🌊"
       }
     },
     {
       "CVDataTitle" : "EmojiCategory-Objects",
       "CVCategoryImage" : "Emoji-ObjectsImage",
       "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "⌚️,📱,📲,💻,⌨️,🖥,🖨,🖱,🖲,🕹,🗜,💽,💾,💿,📀,📼,📷,📸,📹,🎥,📽,🎞,📞,☎️,📟,📠,📺,📻,🎙,🎚,🎛,⏱,⏲,⏰,🕰,⌛️,⏳,📡,🔋,🔌,💡,🔦,🕯,🗑,🛢,💸,💵,💴,💶,💷,💰,💳,💎,⚖️,🔧,🔨,⚒,🛠,⛏,🔩,⚙️,⛓,🔫,💣,🔪,🗡,⚔️,🛡,🚬,⚰️,⚱️,🏺,🔮,📿,💈,⚗️,🔭,🔬,🕳,💊,💉,🌡,🚽,🚰,🚿,🛁,🛀,🛎,🔑,🗝,🚪,🛋,🛏,🛌,🖼,🛍,🛒,🎁,🎈,🎏,🎀,🎊,🎉,🎎,🏮,🎐,✉️,📩,📨,📧,💌,📥,📤,📦,🏷,📪,📫,📬,📭,📮,📯,📜,📃,📄,📑,📊,📈,📉,🗒,🗓,📆,📅,📇,🗃,🗳,🗄,📋,📁,📂,🗂,🗞,📰,📓,📔,📒,📕,📗,📘,📙,📚,📖,🔖,🔗,📎,🖇,📐,📏,📌,📍,✂️,🖊,🖋,✒️,🖌,🖍,📝,✏️,🔍,🔎,🔏,🔐,🔒,🔓"
+        "CVSkipNullGlyphs" : true,
+        "Data" : "🎍,💝,🎎,🎒,🎓,🎏,🎆,🎇,🎐,🎑,🎃,👻,🎅,🎄,🎁,🎋,🎉,🎊,🎈,🎌,🔮,🎥,📷,📹,📼,💿,📀,💽,💾,💻,📱,☎️,📞,📟,📠,📡,📺,📻,🔊,🔉,🔈,🔇,🔔,🔕,📢,📣,⏳,⌛️,⏰,⌚️,🔓,🔒,🔏,🔐,🔑,🔎,💡,🔦,🔆,🔅,🔌,🔋,🔍,🛁,🛀,🚿,🚽,🔧,🔩,🔨,🚪,🚬,💣,🔫,🔪,💊,💉,💰,💴,💵,💷,💶,💳,💸,📲,📧,📥,📤,✉️,📩,📨,📯,📫,📪,📬,📭,📮,📦,📝,📄,📃,📑,📊,📈,📉,📜,📋,📅,📆,📇,📁,📂,✂️,📌,📎,✒️,✏️,📏,📐,📕,📗,📘,📙,📓,📔,📒,📚,📖,🔖,📛,🔬,🔭,📰,🎨,🎬,🎤,🎧,🎼,🎵,🎶,🎹,🎻,🎺,🎷,🎸,👾,🎮,🃏,🎴,🀄️,🎲,🎯,🏈,🏀,⚽️,⚾️,🎾,🎱,🏉,🎳,⛳️,🚵,🚴,🏁,🏇,🏆,🎿,🏂,🏊,🏄,🎣,☕️,🍵,🍶,🍼,🍺,🍻,🍸,🍹,🍷,🍴,🍕,🍔,🍟,🍗,🍖,🍝,🍛,🍤,🍱,🍣,🍥,🍙,🍘,🍚,🍜,🍲,🍢,🍡,🍳,🍞,🍩,🍮,🍦,🍨,🍧,🎂,🍰,🍪,🍫,🍬,🍭,🍯,🍎,🍏,🍊,🍋,🍒,🍇,🍉,🍓,🍑,🍈,🍌,🍐,🍍,🍠,🍆,🍅,🌽"
+      }
+    },
+    {
+      "CVDataTitle" : "EmojiCategory-Places",
+      "CVCategoryImage" : "Emoji-PlacesImage",
+      "CVCategoryData" : {
+        "CVSkipNullGlyphs" : true,
+        "Data" : "🏠,🏡,🏫,🏢,🏣,🏥,🏦,🏪,🏩,🏨,💒,⛪️,🏬,🏤,🌇,🌆,🏯,🏰,⛺️,🏭,🗼,🗾,🗻,🌄,🌅,🌃,🗽,🌉,🎠,🎡,⛲️,🎢,🚢,⛵️,🚤,🚣,⚓️,🚀,✈️,💺,🚁,🚂,🚊,🚉,🚞,🚆,🚄,🚅,🚈,🚇,🚝,🚋,🚃,🚎,🚌,🚍,🚙,🚘,🚗,🚕,🚖,🚛,🚚,🚨,🚓,🚔,🚒,🚑,🚐,🚲,🚡,🚟,🚠,🚜,💈,🚏,🎫,🚦,🚥,⚠️,🚧,🔰,⛽️,🏮,🎰,♨️,🗿,🎪,🎭,📍,🚩,🇯🇵,🇰🇷,🇩🇪,🇨🇳,🇺🇸,🇫🇷,🇪🇸,🇮🇹,🇷🇺,🇬🇧"
       }
     },
     {
       "CVDataTitle" : "EmojiCategory-Symbols",
       "CVCategoryImage" : "Emoji-SymbolImage",
       "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "❤️,💛,💚,💙,💜,🖤,💔,❣️,💕,💞,💓,💗,💖,💘,💝,💟,☮️,✝️,☪️,🕉,☸️,✡️,🔯,🕎,☯️,☦️,🛐,⛎,♈️,♉️,♊️,♋️,♌️,♍️,♎️,♏️,♐️,♑️,♒️,♓️,🆔,⚛️,🉑,☢️,☣️,📴,📳,🈶,🈚️,🈸,🈺,🈷️,✴️,🆚,💮,🉐,㊙️,㊗️,🈴,🈵,🈹,🈲,🅰️,🅱️,🆎,🆑,🅾️,🆘,❌,⭕️,🛑,⛔️,📛,🚫,💯,💢,♨️,🚷,🚯,🚳,🚱,🔞,📵,🚭,❗️,❕,❓,❔,‼️,⁉️,🔅,🔆,〽️,⚠️,🚸,🔱,⚜️,🔰,♻️,✅,🈯️,💹,❇️,✳️,❎,🌐,💠,Ⓜ️,🌀,💤,🏧,🚾,♿️,🅿️,🈳,🈂️,🛂,🛃,🛄,🛅,🚹,🚺,🚼,🚻,🚮,🎦,📶,🈁,🔣,ℹ️,🔤,🔡,🔠,🆖,🆗,🆙,🆒,🆕,🆓,0️⃣,1️⃣,2️⃣,3️⃣,4️⃣,5️⃣,6️⃣,7️⃣,8️⃣,9️⃣,🔟,🔢,#️⃣,*️⃣,▶️,⏸,⏯,⏹,⏺,⏭,⏮,⏩,⏪,⏫,⏬,◀️,🔼,🔽,➡️,⬅️,⬆️,⬇️,↗️,↘️,↙️,↖️,↕️,↔️,↪️,↩️,⤴️,⤵️,🔀,🔁,🔂,🔄,🔃,🎵,🎶,➕,➖,➗,✖️,💲,💱,™️,©️,®️,〰️,➰,➿,🔚,🔙,🔛,🔝,🔜,✔️,☑️,🔘,⚪️,⚫️,🔴,🔵,🔺,🔻,🔸,🔹,🔶,🔷,🔳,🔲,▪️,▫️,◾️,◽️,◼️,◻️,⬛️,⬜️,🔈,🔇,🔉,🔊,🔔,🔕,📣,📢,👁‍🗨,💬,💭,🗯,♠️,♣️,♥️,♦️,🃏,🎴,🀄️,🕐,🕑,🕒,🕓,🕔,🕕,🕖,🕗,🕘,🕙,🕚,🕛,🕜,🕝,🕞,🕟,🕠,🕡,🕢,🕣,🕤,🕥,🕦,🕧"
-      }
-    },
-    {
-      "CVDataTitle" : "EmojiCategory-Flags",
-      "CVCategoryImage" : "Emoji-FlagsImage",
-      "CVCategoryData" : {
-        "CVSkipNullGlyphs" : false,
-        "Data" : "🏳️,🏴,🏁,🚩,🏳️‍🌈,🇦🇫,🇦🇽,🇦🇱,🇩🇿,🇦🇸,🇦🇩,🇦🇴,🇦🇮,🇦🇶,🇦🇬,🇦🇷,🇦🇲,🇦🇼,🇦🇺,🇦🇹,🇦🇿,🇧🇸,🇧🇭,🇧🇩,🇧🇧,🇧🇾,🇧🇪,🇧🇿,🇧🇯,🇧🇲,🇧🇹,🇧🇴,🇧🇶,🇧🇦,🇧🇼,🇧🇷,🇮🇴,🇻🇬,🇧🇳,🇧🇬,🇧🇫,🇧🇮,🇨🇻,🇰🇭,🇨🇲,🇨🇦,🇮🇨,🇰🇾,🇨🇫,🇹🇩,🇨🇱,🇨🇳,🇨🇽,🇨🇨,🇨🇴,🇰🇲,🇨🇬,🇨🇩,🇨🇰,🇨🇷,🇨🇮,🇭🇷,🇨🇺,🇨🇼,🇨🇾,🇨🇿,🇩🇰,🇩🇯,🇩🇲,🇩🇴,🇪🇨,🇪🇬,🇸🇻,🇬🇶,🇪🇷,🇪🇪,🇪🇹,🇪🇺,🇫🇰,🇫🇴,🇫🇯,🇫🇮,🇫🇷,🇬🇫,🇵🇫,🇹🇫,🇬🇦,🇬🇲,🇬🇪,🇩🇪,🇬🇭,🇬🇮,🇬🇷,🇬🇱,🇬🇩,🇬🇵,🇬🇺,🇬🇹,🇬🇬,🇬🇳,🇬🇼,🇬🇾,🇭🇹,🇭🇳,🇭🇰,🇭🇺,🇮🇸,🇮🇳,🇮🇩,🇮🇷,🇮🇶,🇮🇪,🇮🇲,🇮🇱,🇮🇹,🇯🇲,🇯🇵,🎌,🇯🇪,🇯🇴,🇰🇿,🇰🇪,🇰🇮,🇽🇰,🇰🇼,🇰🇬,🇱🇦,🇱🇻,🇱🇧,🇱🇸,🇱🇷,🇱🇾,🇱🇮,🇱🇹,🇱🇺,🇲🇴,🇲🇰,🇲🇬,🇲🇼,🇲🇾,🇲🇻,🇲🇱,🇲🇹,🇲🇭,🇲🇶,🇲🇷,🇲🇺,🇾🇹,🇲🇽,🇫🇲,🇲🇩,🇲🇨,🇲🇳,🇲🇪,🇲🇸,🇲🇦,🇲🇿,🇲🇲,🇳🇦,🇳🇷,🇳🇵,🇳🇱,🇳🇨,🇳🇿,🇳🇮,🇳🇪,🇳🇬,🇳🇺,🇳🇫,🇲🇵,🇰🇵,🇳🇴,🇴🇲,🇵🇰,🇵🇼,🇵🇸,🇵🇦,🇵🇬,🇵🇾,🇵🇪,🇵🇭,🇵🇳,🇵🇱,🇵🇹,🇵🇷,🇶🇦,🇷🇪,🇷🇴,🇷🇺,🇷🇼,🇧🇱,🇸🇭,🇰🇳,🇱🇨,🇵🇲,🇻🇨,🇼🇸,🇸🇲,🇸🇹,🇸🇦,🇸🇳,🇷🇸,🇸🇨,🇸🇱,🇸🇬,🇸🇽,🇸🇰,🇸🇮,🇸🇧,🇸🇴,🇿🇦,🇬🇸,🇰🇷,🇸🇸,🇪🇸,🇱🇰,🇸🇩,🇸🇷,🇸🇿,🇸🇪,🇨🇭,🇸🇾,🇹🇼,🇹🇯,🇹🇿,🇹🇭,🇹🇱,🇹🇬,🇹🇰,🇹🇴,🇹🇹,🇹🇳,🇹🇷,🇹🇲,🇹🇨,🇹🇻,🇺🇬,🇺🇦,🇦🇪,🇬🇧,🇺🇸,🇻🇮,🇺🇾,🇺🇿,🇻🇺,🇻🇦,🇻🇪,🇻🇳,🇼🇫,🇪🇭,🇾🇪,🇿🇲,🇿🇼"
+        "CVSkipNullGlyphs" : true,
+        "Data" : "1️⃣,2️⃣,3️⃣,4️⃣,5️⃣,6️⃣,7️⃣,8️⃣,9️⃣,0️⃣,🔟,🔢,#️⃣,🔣,⬆️,⬇️,⬅️,➡️,🔠,🔡,🔤,↗️,↖️,↘️,↙️,↔️,↕️,🔄,◀️,▶️,🔼,🔽,↩️,↪️,ℹ️,⏪,⏩,⏫,⏬,⤵️,⤴️,🆗,🔀,🔁,🔂,🆕,🆙,🆒,🆓,🆖,📶,🎦,🈁,🈯️,🈳,🈵,🈴,🈲,🉐,🈹,🈺,🈶,🈚️,🚻,🚹,🚺,🚼,🚾,🚰,🚮,🅿️,♿️,🚭,🈷,🈸,🈂,Ⓜ️,🛂,🛄,🛅,🛃,🉑,㊙️,㊗️,🆑,🆘,🆔,🚫,🔞,📵,🚯,🚱,🚳,🚷,🚸,⛔️,✳️,❇️,❎,✅,✴️,💟,🆚,📳,📴,🅰,🅱,🆎,🅾,💠,➿,♻️,♈️,♉️,♊️,♋️,♌️,♍️,♎️,♏️,♐️,♑️,♒️,♓️,⛎,🔯,🏧,💹,💲,💱,©,®,™,❌,‼️,⁉️,❗️,❓,❕,❔,⭕️,🔝,🔚,🔙,🔛,🔜,🔃,🕛,🕧,🕐,🕜,🕑,🕝,🕒,🕞,🕓,🕟,🕔,🕠,🕕,🕖,🕗,🕘,🕙,🕚,🕡,🕢,🕣,🕤,🕥,🕦,✖️,➕,➖,➗,♠️,♥️,♣️,♦️,💮,💯,✔️,☑️,🔘,🔗,➰,〰,〽️,🔱,◼️,◻️,◾️,◽️,▪️,▫️,🔺,🔲,🔳,⚫️,⚪️,🔴,🔵,🔻,⬜️,⬛️,🔶,🔷,🔸,🔹"
       }
     }
   ],

db/dump.rb

@@ -1,57 +1,28 @@
 require 'emoji'
 require 'json'
-require 'rexml/document'
+
+names_list = File.expand_path('../NamesList.txt', __FILE__)
 
 class UnicodeCharacter
-  attr_reader :code, :description, :version, :aliases
+  attr_reader :code, :description, :aliases
 
-  class CharListener
-    CHAR_TAG = "char".freeze
+  @index = {}
+  class << self
+    attr_reader :index
 
-    def self.parse(io, &block)
-      REXML::Document.parse_stream(io, self.new(&block))
+    def fetch(code, *args, &block)
+      code = code.to_s(16).rjust(4, '0') if code.is_a?(Integer)
+      index.fetch(code, *args, &block)
     end
-
-    def initialize(&block)
-      @callback = block
-    end
-
-    def tag_start(name, attributes)
-      if CHAR_TAG == name
-        @callback.call(
-          attributes.fetch("cp") { return },
-          attributes.fetch("na") { return },
-          attributes.fetch("age", nil),
-        )
-      end
-    end
-
-    def method_missing(*) end
   end
 
-  def self.index
-    return @index if defined? @index
-    @index = {}
-    File.open(File.expand_path('../ucd.nounihan.grouped.xml', __FILE__)) do |source|
-      CharListener.parse(source) do |char, desc, age|
-        uc = UnicodeCharacter.new(char, desc, age)
-        @index[uc.code] = uc
-      end
-    end
-    @index
-  end
-
-  def self.fetch(code)
-    code = code.to_s(16).rjust(4, '0') if code.is_a?(Integer)
-    self.index.fetch(code)
-  end
-
-  def initialize(code, description, version)
+  def initialize(code, description)
     @code = code.downcase
     @description = description.downcase
-    @version = version
     @aliases = []
     @references = []
+
+    self.class.index[@code] = self
   end
 
   def add_alias(string)
@@ -63,74 +34,39 @@ class UnicodeCharacter
   end
 end
 
-unless $stdin.tty?
-  codepoints = STDIN.read.chomp.codepoints.map { |code|
-    UnicodeCharacter.fetch(code)
-  }
-  codepoints.each do |char|
-    printf "%5s: %s", char.code.upcase, char.description
-    printf " (%s)", char.version if char.version
-    puts
+char = nil
+
+File.foreach(names_list) do |line|
+  case line
+  when /^[A-F0-9]{4,5}\t/
+    code, desc = line.chomp.split("\t", 2)
+    codepoint = code.hex
+    char = UnicodeCharacter.new(code, desc)
+  when /^\t= /
+    char.add_alias($')
+  when /^\tx .+ - ([A-F0-9]{4,5})\)$/
+    char.add_reference($1)
   end
-  exit
 end
 
 trap(:PIPE) { abort }
 
-normalize = -> (raw) {
-  raw.sub(Emoji::VARIATION_SELECTOR_16, '')
-}
-
-emojidesc = {}
-File.open(File.expand_path('../emoji-test.txt', __FILE__)) do |file|
-  file.each do |line|
-    next if line =~ /^(#|$)/
-    line = line.chomp.split('# ', 2)[1]
-    emoji, description = line.split(' ', 2)
-    emojidesc[normalize.(emoji)] = description
-  end
-end
-
 items = []
+variation_codepoint = Emoji::VARIATION_SELECTOR_16.codepoints[0]
 
-for category, emojis in Emoji.apple_palette
-  for raw in emojis
-    emoji = Emoji.find_by_unicode(raw)
-    unicode_version = emoji ? emoji.unicode_version : ''
-    ios_version = emoji ? emoji.ios_version : ''
+for emoji in Emoji.all
+  item = {}
 
-    unless raw.include?(Emoji::ZERO_WIDTH_JOINER)
-      uchar = UnicodeCharacter.fetch(raw.codepoints[0])
-      unicode_version = uchar.version unless uchar.version.nil?
-    end
-
-    description = emojidesc.fetch(normalize.(raw))
-
-    if unicode_version == ''
-      warn "#{description} (#{raw}) doesn't have Unicode version"
-    end
-
-    if ios_version == ''
-      ios_version = '10.2'
-    end
-
-    items << {
-      emoji: raw,
-      description: description,
-      category: category,
-      aliases: emoji ? emoji.aliases : [description.gsub(/\W+/, '_').downcase],
-      tags: emoji ? emoji.tags : [],
-      unicode_version: unicode_version,
-      ios_version: ios_version,
-    }
+  unless emoji.custom?
+    chars = emoji.raw.codepoints.map { |code| UnicodeCharacter.fetch(code) unless code == variation_codepoint }.compact
+    item[:emoji] = emoji.raw
+    item[:description] = chars.map(&:description).join(' + ')
   end
-end
 
-for emoji in Emoji.all.select(&:custom?)
-  items << {
-    aliases: emoji.aliases,
-    tags: emoji.tags,
-  }
+  item[:aliases] = emoji.aliases
+  item[:tags] = emoji.tags
+
+  items << item
 end
 
 puts JSON.pretty_generate(items)

ext/gemoji/emoji.h

@@ -0,0 +1,11 @@
+static const long emoji_byte_lengths[][5] = {
+	{0},
+	{11, 8, 7, 4, 0},
+	{6, 3, 0},
+	{7, 0},
+	{6, 0},
+	{5, 0},
+};
+
+static const int8_t emoji_magic_bytes[] =
+{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

ext/gemoji/extconf.rb

@@ -0,0 +1,6 @@
+require 'mkmf'
+
+$CFLAGS << ' -ggdb3 -O0 '
+
+dir_config('gemoji')
+create_makefile('gemoji')

ext/gemoji/gemoji.c

@@ -0,0 +1,125 @@
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include "emoji.h"
+
+#define unlikely(x) __builtin_expect((x),0)
+
+static long
+lookup_emoji(VALUE *rb_emoji, VALUE rb_bytes, VALUE rb_unicode_map, const long *possible_len)
+{
+	const uint8_t *src = (uint8_t *)RSTRING_PTR(rb_bytes);
+
+	for (; *possible_len; ++possible_len) {
+		const long emoji_size = *possible_len;
+
+		if (emoji_size > RSTRING_LEN(rb_bytes))
+			continue;
+
+		if ((src[emoji_size - 1] & 0xC0) != 0x80)
+			continue;
+
+		rb_str_set_len(rb_bytes, emoji_size);
+
+		*rb_emoji = rb_hash_lookup(rb_unicode_map, rb_bytes);
+		if (!NIL_P(*rb_emoji))
+			return emoji_size;
+	}
+
+	return 0;
+}
+
+static VALUE
+replace_emoji(const uint8_t *src, long size, VALUE rb_unicode_map)
+{
+	VALUE rb_emoji, rb_bytes, rb_out = Qnil;
+	long i = 0, org, emoji_len;
+	int8_t emoji_byte;
+
+	while (i < size) {
+		org = i;
+
+retry_search:
+		while (i < size && (emoji_byte = emoji_magic_bytes[(int)src[i]]) == 0)
+			i++;
+
+		if (i + 1 < size && (src[i + 1] & 0x80) != 0x80) {
+			i++;
+			goto retry_search;
+		}
+
+		if (unlikely(org == 0)) {
+			if (i == size)
+				return Qnil;
+
+			rb_out = rb_str_buf_new(size * 4 / 3);
+			rb_enc_associate(rb_out, rb_utf8_encoding());
+
+			rb_bytes = rb_str_buf_new(16);
+			rb_enc_associate(rb_bytes, rb_utf8_encoding());
+		}
+
+		if (i > org)
+			rb_str_buf_cat(rb_out, (const char *)src + org, i - org);
+
+		if (unlikely(i == size))
+			break;
+
+		emoji_len = size - i;
+		if (emoji_len > 12)
+			emoji_len = 12;
+
+		memcpy(RSTRING_PTR(rb_bytes), src + i, emoji_len);
+		rb_str_set_len(rb_bytes, emoji_len);
+
+		emoji_len = lookup_emoji(&rb_emoji, rb_bytes, rb_unicode_map, emoji_byte_lengths[(int)emoji_byte]);
+
+		if (emoji_len) {
+			VALUE rb_repl = rb_yield(rb_emoji);
+
+			if (NIL_P(rb_repl)) {
+				rb_str_buf_cat(rb_out, (const char *)src + i, emoji_len);
+			} else {
+				Check_Type(rb_repl, T_STRING);
+				rb_str_buf_append(rb_out, rb_repl);
+			}
+
+			i += emoji_len;
+			continue;
+		}
+
+		rb_str_buf_cat(rb_out, (const char *)src + i, 1);
+		i++;
+	}
+
+	return rb_out;
+}
+
+static VALUE
+rb_gemoji_replace_unicode(VALUE klass, VALUE rb_source)
+{
+	VALUE rb_output;
+	VALUE rb_unicode_map = rb_funcall(klass, rb_intern("unicodes_index"), 0);
+
+	Check_Type(rb_source, T_STRING);
+	Check_Type(rb_unicode_map, T_HASH);
+
+	rb_must_asciicompat(rb_source);
+
+	if (ENC_CODERANGE_ASCIIONLY(rb_source))
+		return rb_source;
+
+	if (rb_enc_get(rb_source) != rb_utf8_encoding())
+		rb_raise(rb_eEncCompatError, "expected UTF-8 encoding");
+
+	rb_output = replace_emoji((uint8_t *)RSTRING_PTR(rb_source), RSTRING_LEN(rb_source), rb_unicode_map);
+	if (NIL_P(rb_output))
+		return rb_source;
+
+	return rb_output;
+}
+
+void Init_gemoji(void)
+{
+	VALUE rb_mEmoji = rb_define_module("Emoji");
+	rb_define_method(rb_mEmoji, "gsub_unicode", rb_gemoji_replace_unicode, 1);
+}

gemoji.gemspec

@@ -1,9 +1,8 @@
 Gem::Specification.new do |s|
   s.name    = "gemoji"
-  s.version = "3.0.0"
-  s.summary = "Emoji library"
-  s.description = "Character information and metadata for standard and custom emoji."
-  s.executables = ["gemoji"]
+  s.version = "2.1.0"
+  s.summary = "Emoji conversion and image assets"
+  s.description = "Image assets and character information for emoji."
 
   s.required_ruby_version = '> 1.9'
 
@@ -14,10 +13,9 @@ Gem::Specification.new do |s|
 
   s.files = Dir[
     "README.md",
-    "bin/gemoji",
-    "images/*.png",
-    "db/Category-Emoji.json",
+    "images/**/*.png",
     "db/emoji.json",
     "lib/**/*.rb",
+    "lib/tasks/*.rake"
   ]
 end