From ec6dca1c16ecb8e0e4f26f425218b19c694a3b6d Mon Sep 17 00:00:00 2001 From: Emi Matchu Date: Wed, 28 Feb 2024 18:52:56 -0800 Subject: [PATCH] Improve Unicode support, emojis don't crash us anymore lol! A few pieces here: 1. Convert all tables to `utf8mb4`+`utf8mb4_unicode_520_ci` strings. 2. Configure that as the server's default. 3. Configure the Rails database connection to use this encoding too. Came together pretty well, whew! This has been a LONG time coming, `latin1` is NOT a good charset for the year 2024! --- config/database.yml | 6 + ...229015410_convert_all_tables_to_utf8mb4.rb | 164 ++++++++++++++++++ db/schema.rb | 80 ++++----- deploy/setup.yml | 12 ++ lib/tasks/db.rake | 48 +++++ 5 files changed, 270 insertions(+), 40 deletions(-) create mode 100644 db/migrate/20240229015410_convert_all_tables_to_utf8mb4.rb create mode 100644 lib/tasks/db.rake diff --git a/config/database.yml b/config/database.yml index b8178043..c2cd0cfb 100644 --- a/config/database.yml +++ b/config/database.yml @@ -8,6 +8,8 @@ development: username: impress_dev password: impress_dev pool: 5 + encoding: utf8mb4 + collation: utf8mb4_unicode_520_ci variables: sql_mode: TRADITIONAL @@ -34,6 +36,8 @@ test: username: impress_dev password: impress_dev pool: 5 + encoding: utf8mb4 + collation: utf8mb4_unicode_520_ci variables: sql_mode: TRADITIONAL @@ -53,6 +57,8 @@ test: production: primary: url: <%= ENV['DATABASE_URL_PRIMARY'] %> + encoding: utf8mb4 + collation: utf8mb4_unicode_520_ci variables: sql_mode: TRADITIONAL diff --git a/db/migrate/20240229015410_convert_all_tables_to_utf8mb4.rb b/db/migrate/20240229015410_convert_all_tables_to_utf8mb4.rb new file mode 100644 index 00000000..3c343ec7 --- /dev/null +++ b/db/migrate/20240229015410_convert_all_tables_to_utf8mb4.rb @@ -0,0 +1,164 @@ +class ConvertAllTablesToUtf8mb4 < ActiveRecord::Migration[7.1] + def change + # NOTE: This migration was generated by our `db:utf8_migration` task! It + # read the list of tables, compared them to the desired charset, and + # generated both SQL to convert to utf8mb4 and to reset back! + reversible do |direction| + direction.up do + execute <<-SQL + ALTER TABLE alt_styles CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE auth_servers CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE campaigns CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE closet_hangers CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE closet_lists CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE colors CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE contributions CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE donation_features CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE donations CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE item_outfit_relationships CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE items CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE login_cookies CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE modeling_logs CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE neopets_connections CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE outfits CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE parents_swf_assets CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE pet_loads CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE pet_states CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE pet_types CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE pets CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE schema_migrations CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE species CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE swf_assets CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE users CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + execute <<-SQL + ALTER TABLE zones CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci + SQL + end + + direction.down do + execute <<-SQL + ALTER TABLE alt_styles CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci + SQL + execute <<-SQL + ALTER TABLE auth_servers CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE campaigns CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE closet_hangers CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE closet_lists CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE colors CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE contributions CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE donation_features CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE donations CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE item_outfit_relationships CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE items CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci + SQL + execute <<-SQL + ALTER TABLE login_cookies CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE modeling_logs CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE neopets_connections CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE outfits CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE parents_swf_assets CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci + SQL + execute <<-SQL + ALTER TABLE pet_loads CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE pet_states CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci + SQL + execute <<-SQL + ALTER TABLE pet_types CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci + SQL + execute <<-SQL + ALTER TABLE pets CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE schema_migrations CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci + SQL + execute <<-SQL + ALTER TABLE species CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE swf_assets CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci + SQL + execute <<-SQL + ALTER TABLE users CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + execute <<-SQL + ALTER TABLE zones CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci + SQL + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index bdd5864d..6c44af7a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,8 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do - create_table "alt_styles", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t| +ActiveRecord::Schema[7.1].define(version: 2024_02_29_015410) do + create_table "alt_styles", charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "species_id", null: false t.integer "color_id", null: false t.integer "body_id", null: false @@ -22,29 +22,29 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["species_id"], name: "index_alt_styles_on_species_id" end - create_table "auth_servers", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "auth_servers", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "short_name", limit: 10, null: false t.string "name", limit: 40, null: false - t.text "icon", size: :medium, null: false - t.text "gateway", size: :medium, null: false + t.text "icon", size: :long, null: false + t.text "gateway", size: :long, null: false t.string "secret", limit: 64, null: false end - create_table "campaigns", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "campaigns", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "progress", default: 0, null: false t.integer "goal", null: false t.boolean "active", null: false t.datetime "created_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false t.boolean "advertised", default: true, null: false - t.text "description", null: false + t.text "description", size: :long, null: false t.string "purpose", default: "our hosting costs this year", null: false t.string "theme_id", default: "hug", null: false - t.text "thanks" + t.text "thanks", size: :long t.string "name" end - create_table "closet_hangers", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "closet_hangers", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "item_id" t.integer "user_id" t.integer "quantity" @@ -60,9 +60,9 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["user_id"], name: "index_closet_hangers_on_user_id" end - create_table "closet_lists", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "closet_lists", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "name" - t.text "description" + t.text "description", size: :long t.integer "user_id" t.boolean "hangers_owned", null: false t.datetime "created_at", precision: nil @@ -71,14 +71,14 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["user_id"], name: "index_closet_lists_on_user_id" end - create_table "colors", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "colors", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.boolean "basic" t.boolean "standard" t.boolean "prank", default: false, null: false t.string "name", null: false end - create_table "contributions", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "contributions", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "contributed_type", limit: 8, null: false t.integer "contributed_id", null: false t.integer "user_id", null: false @@ -87,14 +87,14 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["user_id"], name: "index_contributions_on_user_id" end - create_table "donation_features", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "donation_features", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "donation_id", null: false t.integer "outfit_id" t.datetime "created_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false end - create_table "donations", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "donations", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "amount", null: false t.string "charge_id", null: false t.integer "user_id" @@ -106,7 +106,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.integer "campaign_id", null: false end - create_table "item_outfit_relationships", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "item_outfit_relationships", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "item_id" t.integer "outfit_id" t.boolean "is_worn" @@ -116,15 +116,15 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["outfit_id", "is_worn"], name: "index_item_outfit_relationships_on_outfit_id_and_is_worn" end - create_table "items", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| - t.text "zones_restrict", null: false - t.text "thumbnail_url", size: :medium, null: false + create_table "items", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| + t.text "zones_restrict", size: :medium, null: false + t.text "thumbnail_url", size: :long, null: false t.string "category", limit: 50 t.string "type", limit: 50 t.integer "rarity_index", limit: 2 t.integer "price", limit: 3, null: false t.integer "weight_lbs", limit: 2 - t.text "species_support_ids", size: :medium + t.text "species_support_ids", size: :long t.datetime "created_at", precision: nil t.datetime "updated_at", precision: nil t.boolean "explicitly_body_specific", default: false, null: false @@ -132,7 +132,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.column "modeling_status_hint", "enum('done','glitchy')" t.boolean "is_manually_nc", default: false, null: false t.string "name", null: false - t.text "description", default: "", null: false + t.text "description", size: :medium, default: "", null: false t.string "rarity", default: "", null: false t.index ["modeling_status_hint", "created_at", "id"], name: "items_modeling_status_hint_and_created_at_and_id" t.index ["modeling_status_hint", "created_at"], name: "items_modeling_status_hint_and_created_at" @@ -140,7 +140,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["modeling_status_hint"], name: "items_modeling_status_hint" end - create_table "login_cookies", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "login_cookies", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "user_id", null: false t.integer "series", null: false t.integer "token", null: false @@ -148,20 +148,20 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["user_id"], name: "login_cookies_user_id" end - create_table "modeling_logs", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "modeling_logs", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.datetime "created_at", precision: nil, default: -> { "current_timestamp()" }, null: false - t.text "log_json", null: false + t.text "log_json", size: :long, null: false t.string "pet_name", limit: 128, null: false end - create_table "neopets_connections", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "neopets_connections", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "user_id" t.string "neopets_username" t.datetime "created_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false end - create_table "outfits", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "outfits", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "pet_state_id" t.integer "user_id" t.datetime "created_at", precision: nil @@ -177,7 +177,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["user_id"], name: "index_outfits_on_user_id" end - create_table "parents_swf_assets", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| + create_table "parents_swf_assets", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "parent_id", limit: 3, null: false t.integer "swf_asset_id", limit: 3, null: false t.string "parent_type", limit: 8, null: false @@ -186,15 +186,15 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["swf_asset_id"], name: "parents_swf_assets_swf_asset_id" end - create_table "pet_loads", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "pet_loads", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "pet_name", limit: 20, null: false - t.text "amf", size: :medium, null: false + t.text "amf", size: :long, null: false t.datetime "created_at", precision: nil, null: false end - create_table "pet_states", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| + create_table "pet_states", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "pet_type_id", limit: 3, null: false - t.text "swf_asset_ids", null: false + t.text "swf_asset_ids", size: :medium, null: false t.boolean "female" t.integer "mood_id" t.boolean "unconverted" @@ -204,7 +204,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["pet_type_id"], name: "pet_states_pet_type_id" end - create_table "pet_types", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| + create_table "pet_types", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "color_id", limit: 1, null: false t.integer "species_id", limit: 1, null: false t.datetime "created_at", precision: nil, null: false @@ -218,23 +218,23 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["species_id", "color_id"], name: "pet_types_species_color", unique: true end - create_table "pets", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "pets", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "name", limit: 20, null: false t.integer "pet_type_id", limit: 3, null: false t.index ["name"], name: "pets_name", unique: true t.index ["pet_type_id"], name: "pets_pet_type_id" end - create_table "species", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "species", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "name", null: false end - create_table "swf_assets", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| + create_table "swf_assets", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "type", limit: 7, null: false t.integer "remote_id", limit: 3, null: false - t.text "url", size: :medium, null: false + t.text "url", size: :long, null: false t.integer "zone_id", limit: 1, null: false - t.text "zones_restrict", null: false + t.text "zones_restrict", size: :medium, null: false t.datetime "created_at", precision: nil, null: false t.integer "body_id", limit: 2, null: false t.boolean "has_image", default: false, null: false @@ -242,7 +242,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.datetime "reported_broken_at", precision: nil t.datetime "converted_at", precision: nil t.boolean "image_manual", default: false, null: false - t.text "manifest", size: :medium + t.text "manifest", size: :long t.timestamp "manifest_cached_at" t.string "known_glitches", limit: 128, default: "" t.string "manifest_url" @@ -253,7 +253,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.index ["zone_id"], name: "idx_swf_assets_zone_id" end - create_table "users", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "users", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.string "name", limit: 20, null: false t.integer "auth_server_id", limit: 1, null: false t.integer "remote_id", null: false @@ -268,7 +268,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do t.boolean "support_staff", default: false, null: false end - create_table "zones", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| + create_table "zones", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t| t.integer "depth" t.integer "type_id" t.string "label", null: false diff --git a/deploy/setup.yml b/deploy/setup.yml index a6d2caa1..2646fbe9 100644 --- a/deploy/setup.yml +++ b/deploy/setup.yml @@ -384,6 +384,18 @@ skip-bind-address notify: Restart MariaDB + # This is the best Unicode collation available in our version of MariaDB! + # We already specify it for all the tables in `schema.rb`, but also set it + # as the default collation for new tables here, too. + - name: Set MariaDb's default collation to utf8mb4_unicode_520_ci + copy: + dest: /etc/mysql/mariadb.conf.d/80-charsets.cnf + content: | + [mysqld] + character-set-server=utf8mb4 + collation-server=utf8mb4_unicode_520_ci + notify: Restart MariaDB + - name: Enable slow query logging for MariaDB copy: dest: /etc/mysql/mariadb.conf.d/80-logging.cnf diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake new file mode 100644 index 00000000..2c193990 --- /dev/null +++ b/lib/tasks/db.rake @@ -0,0 +1,48 @@ +namespace :db do + desc "Generate SQL to convert all tables and columns to utf8mb4" + task :utf8_migration => :environment do + target_charset = "utf8mb4" + target_collation = "utf8mb4_unicode_520_ci" + + db = ApplicationRecord.connection + + # Don't mess with ActiveRecord's internal tables! + tables = db.tables.reject { |t| t.start_with? "ar_internal_" } + + ups = [] + downs = [] + tables.sort.each do |table| + # If the table's default charset/collation doesn't match our goal, we'll + # modify it! + db.table_options(table) => {charset:, collation:} + if charset != target_charset || collation != target_collation + ups << "ALTER TABLE #{table} CONVERT TO CHARACTER SET " + + "#{target_charset} COLLATE #{target_collation}" + downs << "ALTER TABLE #{table} CONVERT TO CHARACTER SET " + + "#{charset} COLLATE #{collation}" + end + end + + puts <<~MIGRATION_RB + reversible do |direction| + direction.up do + #{sql_as_ruby(ups).indent(2, "\t")} + end + + direction.down do + #{sql_as_ruby(downs).indent(2, "\t")} + end + end + MIGRATION_RB + end +end + +def sql_as_ruby(statements) + statements.map do |sql| + <<~EXECUTE_RB + execute <<-SQL + #{sql} + SQL + EXECUTE_RB + end.join("") +end