Merge pull request #414 from knu/set_charset_for_mysql

Set charset/collation properly for each text column if using MySQL.

Gemfile.lock

@@ -45,9 +45,10 @@ GEM
       extlib (>= 0.9.15)
       multi_json (>= 1.0.0)
     bcrypt (3.1.7)
-    better_errors (1.1.0)
+    better_errors (2.0.0)
       coderay (>= 1.0.0)
       erubis (>= 2.6.6)
+      rack (>= 0.9.0)
     binding_of_caller (0.7.2)
       debug_inspector (>= 0.0.1)
     bootstrap-kaminari-views (0.0.3)

config/initializers/ar_mysql_column_charset.rb

@@ -0,0 +1,3 @@
+ActiveSupport.on_load :active_record do
+  require 'ar_mysql_column_charset'
+end

db/migrate/20140813110107_set_charset_for_mysql.rb

@@ -0,0 +1,74 @@
+class SetCharsetForMysql < ActiveRecord::Migration
+  def all_models
+    @all_models ||= [
+      Agent,
+      AgentLog,
+      Contact,
+      Event,
+      Link,
+      Scenario,
+      ScenarioMembership,
+      User,
+      UserCredential,
+      Delayed::Job,
+    ]
+  end
+
+  def change
+    conn = ActiveRecord::Base.connection
+
+    # This is migration is for MySQL only.
+    return unless conn.is_a?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter)
+
+    reversible do |dir|
+      dir.up do
+        all_models.each { |model|
+          table_name = model.table_name
+
+          # `contacts` may not exist
+          next unless connection.table_exists? table_name
+
+          model.columns.each { |column|
+            name = column.name
+            type = column.type
+            limit = column.limit
+            options = {
+              limit: limit,
+              null: column.null,
+              default: column.default,
+            }
+
+            case type
+            when :string, :text
+              options.update(charset: 'utf8', collation: 'utf8_unicode_ci')
+              case name
+              when 'username'
+                options.update(limit: 767 / 4, charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci')
+              when 'message', 'options', 'name', 'memory',
+                   'handler', 'last_error', 'payload', 'description'
+                options.update(charset: 'utf8mb4', collation: 'utf8mb4_bin')
+              when 'type', 'schedule', 'mode', 'email',
+                   'invitation_code', 'reset_password_token'
+                options.update(collation: 'utf8_bin')
+              when 'guid', 'encrypted_password'
+                options.update(charset: 'ascii', collation: 'ascii_bin')
+              end
+            else
+              next
+            end
+
+            change_column table_name, name, type, options
+          }
+
+          execute 'ALTER TABLE %s CHARACTER SET utf8 COLLATE utf8_unicode_ci' % table_name
+        }
+
+        execute 'ALTER DATABASE %s CHARACTER SET utf8 COLLATE utf8_unicode_ci' % conn.current_database
+      end
+
+      dir.down do
+        # Do nada; no use to go back
+      end
+    end
+  end
+end

db/schema.rb

@@ -18,7 +18,7 @@ ActiveRecord::Schema.define(version: 20140820003139) do
 
   create_table "agent_logs", force: true do |t|
     t.integer  "agent_id",                      null: false
-    t.text     "message",                       null: false
+    t.text     "message",           limit: 16777215,             null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
     t.integer  "level",             default: 3, null: false
     t.integer  "inbound_event_id"
     t.integer  "outbound_event_id"
@@ -28,24 +28,24 @@ ActiveRecord::Schema.define(version: 20140820003139) do
 
   create_table "agents", force: true do |t|
     t.integer  "user_id"
-    t.text     "options"
-    t.string   "type"
-    t.string   "name"
-    t.string   "schedule"
+    t.text     "options",               limit: 16777215,                                charset: "utf8mb4", collation: "utf8mb4_bin"
+    t.string   "type",                                                                                      collation: "utf8_bin"
+    t.string   "name",                                                                  charset: "utf8mb4", collation: "utf8mb4_bin"
+    t.string   "schedule",                                                                                  collation: "utf8_bin"
     t.integer  "events_count"
     t.datetime "last_check_at"
     t.datetime "last_receive_at"
     t.integer  "last_checked_event_id"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.text     "memory"
+    t.datetime "created_at",                                               null: false
+    t.datetime "updated_at",                                               null: false
+    t.text     "memory",                limit: 2147483647,                              charset: "utf8mb4", collation: "utf8mb4_bin"
     t.datetime "last_web_request_at"
     t.integer  "keep_events_for",       default: 0,     null: false
     t.datetime "last_event_at"
     t.datetime "last_error_log_at"
     t.boolean  "propagate_immediately", default: false, null: false
     t.boolean  "disabled",              default: false, null: false
-    t.string   "guid",                                  null: false
+    t.string   "guid",                                                     null: false, charset: "ascii",   collation: "ascii_bin"
     t.integer  "service_id"
   end
 
@@ -55,10 +55,10 @@ ActiveRecord::Schema.define(version: 20140820003139) do
   add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree
 
   create_table "delayed_jobs", force: true do |t|
-    t.integer  "priority",   default: 0
-    t.integer  "attempts",   default: 0
-    t.text     "handler"
-    t.text     "last_error"
+    t.integer  "priority",                    default: 0
+    t.integer  "attempts",                    default: 0
+    t.text     "handler",    limit: 16777215,                          charset: "utf8mb4", collation: "utf8mb4_bin"
+    t.text     "last_error", limit: 16777215,                          charset: "utf8mb4", collation: "utf8mb4_bin"
     t.datetime "run_at"
     t.datetime "locked_at"
     t.datetime "failed_at"
@@ -73,11 +73,11 @@ ActiveRecord::Schema.define(version: 20140820003139) do
   create_table "events", force: true do |t|
     t.integer  "user_id"
     t.integer  "agent_id"
-    t.decimal  "lat",        precision: 15, scale: 10
-    t.decimal  "lng",        precision: 15, scale: 10
-    t.text     "payload"
-    t.datetime "created_at"
-    t.datetime "updated_at"
+    t.decimal  "lat",                           precision: 15, scale: 10
+    t.decimal  "lng",                           precision: 15, scale: 10
+    t.text     "payload",    limit: 2147483647,                                        charset: "utf8mb4", collation: "utf8mb4_bin"
+    t.datetime "created_at",                                              null: false
+    t.datetime "updated_at",                                              null: false
     t.datetime "expires_at"
   end
 
@@ -107,13 +107,13 @@ ActiveRecord::Schema.define(version: 20140820003139) do
   add_index "scenario_memberships", ["scenario_id"], name: "index_scenario_memberships_on_scenario_id", using: :btree
 
   create_table "scenarios", force: true do |t|
-    t.string   "name",                         null: false
-    t.integer  "user_id",                      null: false
+    t.string   "name",                        null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
+    t.integer  "user_id",                     null: false
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.text     "description"
-    t.boolean  "public",       default: false, null: false
-    t.string   "guid",                         null: false
+    t.text     "description",                              charset: "utf8mb4", collation: "utf8mb4_bin"
+    t.boolean  "public",      default: false, null: false
+    t.string   "guid",                        null: false, charset: "ascii",   collation: "ascii_bin"
     t.string   "source_url"
     t.string   "tag_bg_color"
     t.string   "tag_fg_color"
@@ -144,33 +144,33 @@ ActiveRecord::Schema.define(version: 20140820003139) do
     t.integer  "user_id",                           null: false
     t.string   "credential_name",                   null: false
     t.text     "credential_value",                  null: false
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string   "mode",             default: "text", null: false
+    t.datetime "created_at",                        null: false
+    t.datetime "updated_at",                        null: false
+    t.string   "mode",             default: "text", null: false, collation: "utf8_bin"
   end
 
   add_index "user_credentials", ["user_id", "credential_name"], name: "index_user_credentials_on_user_id_and_credential_name", unique: true, using: :btree
 
   create_table "users", force: true do |t|
-    t.string   "email",                  default: "",    null: false
-    t.string   "encrypted_password",     default: "",    null: false
-    t.string   "reset_password_token"
+    t.string   "email",                              default: "",    null: false,                     collation: "utf8_bin"
+    t.string   "encrypted_password",                 default: "",    null: false, charset: "ascii",   collation: "ascii_bin"
+    t.string   "reset_password_token",                                                                collation: "utf8_bin"
     t.datetime "reset_password_sent_at"
     t.datetime "remember_created_at"
-    t.integer  "sign_in_count",          default: 0
+    t.integer  "sign_in_count",                      default: 0
     t.datetime "current_sign_in_at"
     t.datetime "last_sign_in_at"
     t.string   "current_sign_in_ip"
     t.string   "last_sign_in_ip"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.boolean  "admin",                  default: false, null: false
-    t.integer  "failed_attempts",        default: 0
+    t.datetime "created_at",                                         null: false
+    t.datetime "updated_at",                                         null: false
+    t.boolean  "admin",                              default: false, null: false
+    t.integer  "failed_attempts",                    default: 0
     t.string   "unlock_token"
     t.datetime "locked_at"
-    t.string   "username",                               null: false
-    t.string   "invitation_code",                        null: false
-    t.integer  "scenario_count",         default: 0,     null: false
+    t.string   "username",               limit: 191,                 null: false, charset: "utf8mb4", collation: "utf8mb4_unicode_ci"
+    t.string   "invitation_code",                                    null: false,                     collation: "utf8_bin"
+    t.integer  "scenario_count",                     default: 0,     null: false
   end
 
   add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree

lib/ar_mysql_column_charset.rb

@@ -0,0 +1,110 @@
+require 'active_record'
+
+# Module#prepend support for Ruby 1.9
+require 'prepend' unless Module.method_defined?(:prepend)
+
+module ActiveRecord::ConnectionAdapters
+  class ColumnDefinition
+    module CharsetSupport
+      attr_accessor :charset, :collation
+    end
+
+    prepend CharsetSupport
+  end
+
+  class TableDefinition
+    module CharsetSupport
+      def new_column_definition(name, type, options)
+        column = super
+        column.charset   = options[:charset]
+        column.collation = options[:collation]
+        column
+      end
+    end
+
+    prepend CharsetSupport
+  end
+
+  class AbstractMysqlAdapter
+    module CharsetSupport
+      def prepare_column_options(column, types)
+        spec = super
+        conn = ActiveRecord::Base.connection
+        spec[:charset]   = column.charset.inspect if column.charset && column.charset != conn.charset
+        spec[:collation] = column.collation.inspect if column.collation && column.collation != conn.collation
+        spec
+      end
+
+      def migration_keys
+        super + [:charset, :collation]
+      end
+
+      def utf8mb4_supported?
+        if @utf8mb4_supported.nil?
+          @utf8mb4_supported = !select("show character set like 'utf8mb4'").empty?
+        else
+          @utf8mb4_supported
+        end
+      end
+
+      def charset_collation(charset, collation)
+        [charset, collation].map { |name|
+          case name
+          when nil
+            nil
+          when /\A(utf8mb4(_\w*)?)\z/
+            if utf8mb4_supported?
+              $1
+            else
+              "utf8#{$2}"
+            end
+          else
+            name.to_s
+          end
+        }
+      end
+    end
+
+    prepend CharsetSupport
+
+    class SchemaCreation
+      module CharsetSupport
+        def column_options(o)
+          column_options = super
+          column_options[:charset]   = o.charset unless o.charset.nil?
+          column_options[:collation] = o.collation unless o.collation.nil?
+          column_options
+        end
+
+        def add_column_options!(sql, options)
+          charset, collation = @conn.charset_collation(options[:charset], options[:collation])
+
+          if charset
+            sql << " CHARACTER SET #{charset}"
+          end
+
+          if collation
+            sql << " COLLATE #{collation}"
+          end
+
+          super
+        end
+      end
+
+      prepend CharsetSupport
+    end
+
+    class Column
+      module CharsetSupport
+        attr_reader :charset
+
+        def initialize(*args)
+          super
+          @charset = @collation[/\A[^_]+/] unless @collation.nil?
+        end
+      end
+
+      prepend CharsetSupport
+    end
+  end
+end

lib/prepend.rb

@@ -0,0 +1,85 @@
+# Fake implementation of prepend(), which does not support overriding
+# inherited methods nor methods that are formerly overridden by
+# another invocation of prepend().
+#
+# Here's what <Original>.prepend(<Wrapper>) does:
+#
+# - Create an anonymous stub module (hereinafter <Stub>) and define
+#   <Stub>#<method> that calls #<method>_without_<Wrapper> for each
+#   instance method of <Wrapper>.
+#
+# - Rename <Original>#<method> to #<method>_without_<Wrapper> for each
+#   instance method of <Wrapper>.
+#
+# - Include <Stub> and <Wrapper> into <Original> in that order.
+#
+# This way, a call of <Original>#<method> is dispatched to
+# <Wrapper><method>, which may call super which is dispatched to
+# <Stub>#<method>, which finally calls
+# <Original>#<method>_without_<Wrapper> which is used to be called
+# <Original>#<method>.
+#
+# Usage:
+#
+#     class Mechanize
+#       # module with methods that overrides those of X
+#       module Y
+#       end
+#
+#       unless X.respond_to?(:prepend, true)
+#         require 'mechanize/prependable'
+#         X.extend(Prependable)
+#       end
+#
+#       class X
+#         prepend Y
+#       end
+#     end
+class Module
+  def prepend(mod)
+    stub = Module.new
+
+    mod_id = (mod.name || 'Module__%d' % mod.object_id).gsub(/::/, '__')
+
+    mod.instance_methods.each { |name|
+      method_defined?(name) or next
+
+      original = instance_method(name)
+      next if original.owner != self
+
+      name = name.to_s
+      name_without = name.sub(/(?=[?!=]?\z)/) { '_without_%s' % mod_id }
+
+      arity = original.arity
+      arglist = (
+        if arity >= 0
+          (1..arity).map { |i| 'x%d' % i }
+        else
+          (1..(-arity - 1)).map { |i| 'x%d' % i } << '*a'
+        end << '&b'
+      ).join(', ')
+
+      if name.end_with?('=')
+        stub.module_eval %{
+          def #{name}(#{arglist})
+            __send__(:#{name_without}, #{arglist})
+          end
+        }
+      else
+        stub.module_eval %{
+          def #{name}(#{arglist})
+            #{name_without}(#{arglist})
+          end
+        }
+      end
+      module_eval {
+        alias_method name_without, name
+        remove_method name
+      }
+    }
+
+    include stub
+    include mod
+  end
+  private :prepend
+end unless Module.method_defined?(:prepend)