OpenNeo/impress
OpenNeo/impress
1
0
Fork 1
Code Pull requests Activity

Improve Unicode support, emojis don't crash us anymore lol!

Browse source 
A few pieces here:

1. Convert all tables to `utf8mb4`+`utf8mb4_unicode_520_ci` strings.
2. Configure that as the server's default.
3. Configure the Rails database connection to use this encoding too.

Came together pretty well, whew! This has been a LONG time coming,
`latin1` is NOT a good charset for the year 2024!
This commit is contained in:
Emi Matchu 2024-02-28 18:52:56 -08:00
parent 8f78892212
commit ec6dca1c16
5 changed files with 270 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config/database.yml
View file

@ -8,6 +8,8 @@ development:
username: impress_dev username: impress_dev
password: impress_dev password: impress_dev
pool: 5 pool: 5
encoding: utf8mb4
collation: utf8mb4_unicode_520_ci
variables: variables:
sql_mode: TRADITIONAL sql_mode: TRADITIONAL
@ -34,6 +36,8 @@ test:
username: impress_dev username: impress_dev
password: impress_dev password: impress_dev
pool: 5 pool: 5
encoding: utf8mb4
collation: utf8mb4_unicode_520_ci
variables: variables:
sql_mode: TRADITIONAL sql_mode: TRADITIONAL
@ -53,6 +57,8 @@ test:
production: production:
primary: primary:
url: <%= ENV['DATABASE_URL_PRIMARY'] %> url: <%= ENV['DATABASE_URL_PRIMARY'] %>
encoding: utf8mb4
collation: utf8mb4_unicode_520_ci
variables: variables:
sql_mode: TRADITIONAL sql_mode: TRADITIONAL

164
db/migrate/20240229015410_convert_all_tables_to_utf8mb4.rb Normal file
View file

@ -0,0 +1,164 @@
class ConvertAllTablesToUtf8mb4 < ActiveRecord::Migration[7.1]
def change
# NOTE: This migration was generated by our `db:utf8_migration` task! It
# read the list of tables, compared them to the desired charset, and
# generated both SQL to convert to utf8mb4 and to reset back!
reversible do |direction|
direction.up do
execute <<-SQL
ALTER TABLE alt_styles CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE auth_servers CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE campaigns CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE closet_hangers CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE closet_lists CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE colors CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE contributions CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE donation_features CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE donations CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE item_outfit_relationships CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE items CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE login_cookies CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE modeling_logs CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE neopets_connections CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE outfits CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE parents_swf_assets CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE pet_loads CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE pet_states CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE pet_types CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE pets CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE schema_migrations CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE species CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE swf_assets CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE users CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
execute <<-SQL
ALTER TABLE zones CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci
SQL
end
direction.down do
execute <<-SQL
ALTER TABLE alt_styles CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
SQL
execute <<-SQL
ALTER TABLE auth_servers CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE campaigns CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE closet_hangers CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE closet_lists CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE colors CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE contributions CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE donation_features CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE donations CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE item_outfit_relationships CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE items CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci
SQL
execute <<-SQL
ALTER TABLE login_cookies CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE modeling_logs CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE neopets_connections CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE outfits CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE parents_swf_assets CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci
SQL
execute <<-SQL
ALTER TABLE pet_loads CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE pet_states CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci
SQL
execute <<-SQL
ALTER TABLE pet_types CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci
SQL
execute <<-SQL
ALTER TABLE pets CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE schema_migrations CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci
SQL
execute <<-SQL
ALTER TABLE species CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE swf_assets CONVERT TO CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci
SQL
execute <<-SQL
ALTER TABLE users CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
execute <<-SQL
ALTER TABLE zones CONVERT TO CHARACTER SET latin1 COLLATE latin1_swedish_ci
SQL
end
end
end
end

80
db/schema.rb
View file

@ -10,8 +10,8 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do ActiveRecord::Schema[7.1].define(version: 2024_02_29_015410) do
create_table "alt_styles", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t| create_table "alt_styles", charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "species_id", null: false t.integer "species_id", null: false
t.integer "color_id", null: false t.integer "color_id", null: false
t.integer "body_id", null: false t.integer "body_id", null: false
@ -22,29 +22,29 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["species_id"], name: "index_alt_styles_on_species_id" t.index ["species_id"], name: "index_alt_styles_on_species_id"
end end
create_table "auth_servers", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "auth_servers", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "short_name", limit: 10, null: false t.string "short_name", limit: 10, null: false
t.string "name", limit: 40, null: false t.string "name", limit: 40, null: false
t.text "icon", size: :medium, null: false t.text "icon", size: :long, null: false
t.text "gateway", size: :medium, null: false t.text "gateway", size: :long, null: false
t.string "secret", limit: 64, null: false t.string "secret", limit: 64, null: false
end end
create_table "campaigns", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "campaigns", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "progress", default: 0, null: false t.integer "progress", default: 0, null: false
t.integer "goal", null: false t.integer "goal", null: false
t.boolean "active", null: false t.boolean "active", null: false
t.datetime "created_at", precision: nil, null: false t.datetime "created_at", precision: nil, null: false
t.datetime "updated_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false
t.boolean "advertised", default: true, null: false t.boolean "advertised", default: true, null: false
t.text "description", null: false t.text "description", size: :long, null: false
t.string "purpose", default: "our hosting costs this year", null: false t.string "purpose", default: "our hosting costs this year", null: false
t.string "theme_id", default: "hug", null: false t.string "theme_id", default: "hug", null: false
t.text "thanks" t.text "thanks", size: :long
t.string "name" t.string "name"
end end
create_table "closet_hangers", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "closet_hangers", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "item_id" t.integer "item_id"
t.integer "user_id" t.integer "user_id"
t.integer "quantity" t.integer "quantity"
@ -60,9 +60,9 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["user_id"], name: "index_closet_hangers_on_user_id" t.index ["user_id"], name: "index_closet_hangers_on_user_id"
end end
create_table "closet_lists", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "closet_lists", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "name" t.string "name"
t.text "description" t.text "description", size: :long
t.integer "user_id" t.integer "user_id"
t.boolean "hangers_owned", null: false t.boolean "hangers_owned", null: false
t.datetime "created_at", precision: nil t.datetime "created_at", precision: nil
@ -71,14 +71,14 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["user_id"], name: "index_closet_lists_on_user_id" t.index ["user_id"], name: "index_closet_lists_on_user_id"
end end
create_table "colors", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "colors", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.boolean "basic" t.boolean "basic"
t.boolean "standard" t.boolean "standard"
t.boolean "prank", default: false, null: false t.boolean "prank", default: false, null: false
t.string "name", null: false t.string "name", null: false
end end
create_table "contributions", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "contributions", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "contributed_type", limit: 8, null: false t.string "contributed_type", limit: 8, null: false
t.integer "contributed_id", null: false t.integer "contributed_id", null: false
t.integer "user_id", null: false t.integer "user_id", null: false
@ -87,14 +87,14 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["user_id"], name: "index_contributions_on_user_id" t.index ["user_id"], name: "index_contributions_on_user_id"
end end
create_table "donation_features", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "donation_features", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "donation_id", null: false t.integer "donation_id", null: false
t.integer "outfit_id" t.integer "outfit_id"
t.datetime "created_at", precision: nil, null: false t.datetime "created_at", precision: nil, null: false
t.datetime "updated_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false
end end
create_table "donations", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "donations", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "amount", null: false t.integer "amount", null: false
t.string "charge_id", null: false t.string "charge_id", null: false
t.integer "user_id" t.integer "user_id"
@ -106,7 +106,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.integer "campaign_id", null: false t.integer "campaign_id", null: false
end end
create_table "item_outfit_relationships", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "item_outfit_relationships", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "item_id" t.integer "item_id"
t.integer "outfit_id" t.integer "outfit_id"
t.boolean "is_worn" t.boolean "is_worn"
@ -116,15 +116,15 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["outfit_id", "is_worn"], name: "index_item_outfit_relationships_on_outfit_id_and_is_worn" t.index ["outfit_id", "is_worn"], name: "index_item_outfit_relationships_on_outfit_id_and_is_worn"
end end
create_table "items", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| create_table "items", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.text "zones_restrict", null: false t.text "zones_restrict", size: :medium, null: false
t.text "thumbnail_url", size: :medium, null: false t.text "thumbnail_url", size: :long, null: false
t.string "category", limit: 50 t.string "category", limit: 50
t.string "type", limit: 50 t.string "type", limit: 50
t.integer "rarity_index", limit: 2 t.integer "rarity_index", limit: 2
t.integer "price", limit: 3, null: false t.integer "price", limit: 3, null: false
t.integer "weight_lbs", limit: 2 t.integer "weight_lbs", limit: 2
t.text "species_support_ids", size: :medium t.text "species_support_ids", size: :long
t.datetime "created_at", precision: nil t.datetime "created_at", precision: nil
t.datetime "updated_at", precision: nil t.datetime "updated_at", precision: nil
t.boolean "explicitly_body_specific", default: false, null: false t.boolean "explicitly_body_specific", default: false, null: false
@ -132,7 +132,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.column "modeling_status_hint", "enum('done','glitchy')" t.column "modeling_status_hint", "enum('done','glitchy')"
t.boolean "is_manually_nc", default: false, null: false t.boolean "is_manually_nc", default: false, null: false
t.string "name", null: false t.string "name", null: false
t.text "description", default: "", null: false t.text "description", size: :medium, default: "", null: false
t.string "rarity", default: "", null: false t.string "rarity", default: "", null: false
t.index ["modeling_status_hint", "created_at", "id"], name: "items_modeling_status_hint_and_created_at_and_id" t.index ["modeling_status_hint", "created_at", "id"], name: "items_modeling_status_hint_and_created_at_and_id"
t.index ["modeling_status_hint", "created_at"], name: "items_modeling_status_hint_and_created_at" t.index ["modeling_status_hint", "created_at"], name: "items_modeling_status_hint_and_created_at"
@ -140,7 +140,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["modeling_status_hint"], name: "items_modeling_status_hint" t.index ["modeling_status_hint"], name: "items_modeling_status_hint"
end end
create_table "login_cookies", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "login_cookies", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "user_id", null: false t.integer "user_id", null: false
t.integer "series", null: false t.integer "series", null: false
t.integer "token", null: false t.integer "token", null: false
@ -148,20 +148,20 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["user_id"], name: "login_cookies_user_id" t.index ["user_id"], name: "login_cookies_user_id"
end end
create_table "modeling_logs", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "modeling_logs", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.datetime "created_at", precision: nil, default: -> { "current_timestamp()" }, null: false t.datetime "created_at", precision: nil, default: -> { "current_timestamp()" }, null: false
t.text "log_json", null: false t.text "log_json", size: :long, null: false
t.string "pet_name", limit: 128, null: false t.string "pet_name", limit: 128, null: false
end end
create_table "neopets_connections", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "neopets_connections", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "user_id" t.integer "user_id"
t.string "neopets_username" t.string "neopets_username"
t.datetime "created_at", precision: nil, null: false t.datetime "created_at", precision: nil, null: false
t.datetime "updated_at", precision: nil, null: false t.datetime "updated_at", precision: nil, null: false
end end
create_table "outfits", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "outfits", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "pet_state_id" t.integer "pet_state_id"
t.integer "user_id" t.integer "user_id"
t.datetime "created_at", precision: nil t.datetime "created_at", precision: nil
@ -177,7 +177,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["user_id"], name: "index_outfits_on_user_id" t.index ["user_id"], name: "index_outfits_on_user_id"
end end
create_table "parents_swf_assets", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| create_table "parents_swf_assets", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "parent_id", limit: 3, null: false t.integer "parent_id", limit: 3, null: false
t.integer "swf_asset_id", limit: 3, null: false t.integer "swf_asset_id", limit: 3, null: false
t.string "parent_type", limit: 8, null: false t.string "parent_type", limit: 8, null: false
@ -186,15 +186,15 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["swf_asset_id"], name: "parents_swf_assets_swf_asset_id" t.index ["swf_asset_id"], name: "parents_swf_assets_swf_asset_id"
end end
create_table "pet_loads", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "pet_loads", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "pet_name", limit: 20, null: false t.string "pet_name", limit: 20, null: false
t.text "amf", size: :medium, null: false t.text "amf", size: :long, null: false
t.datetime "created_at", precision: nil, null: false t.datetime "created_at", precision: nil, null: false
end end
create_table "pet_states", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| create_table "pet_states", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "pet_type_id", limit: 3, null: false t.integer "pet_type_id", limit: 3, null: false
t.text "swf_asset_ids", null: false t.text "swf_asset_ids", size: :medium, null: false
t.boolean "female" t.boolean "female"
t.integer "mood_id" t.integer "mood_id"
t.boolean "unconverted" t.boolean "unconverted"
@ -204,7 +204,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["pet_type_id"], name: "pet_states_pet_type_id" t.index ["pet_type_id"], name: "pet_states_pet_type_id"
end end
create_table "pet_types", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| create_table "pet_types", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "color_id", limit: 1, null: false t.integer "color_id", limit: 1, null: false
t.integer "species_id", limit: 1, null: false t.integer "species_id", limit: 1, null: false
t.datetime "created_at", precision: nil, null: false t.datetime "created_at", precision: nil, null: false
@ -218,23 +218,23 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["species_id", "color_id"], name: "pet_types_species_color", unique: true t.index ["species_id", "color_id"], name: "pet_types_species_color", unique: true
end end
create_table "pets", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "pets", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "name", limit: 20, null: false t.string "name", limit: 20, null: false
t.integer "pet_type_id", limit: 3, null: false t.integer "pet_type_id", limit: 3, null: false
t.index ["name"], name: "pets_name", unique: true t.index ["name"], name: "pets_name", unique: true
t.index ["pet_type_id"], name: "pets_pet_type_id" t.index ["pet_type_id"], name: "pets_pet_type_id"
end end
create_table "species", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "species", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "name", null: false t.string "name", null: false
end end
create_table "swf_assets", id: :integer, charset: "utf8mb3", collation: "utf8mb3_unicode_ci", force: :cascade do |t| create_table "swf_assets", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "type", limit: 7, null: false t.string "type", limit: 7, null: false
t.integer "remote_id", limit: 3, null: false t.integer "remote_id", limit: 3, null: false
t.text "url", size: :medium, null: false t.text "url", size: :long, null: false
t.integer "zone_id", limit: 1, null: false t.integer "zone_id", limit: 1, null: false
t.text "zones_restrict", null: false t.text "zones_restrict", size: :medium, null: false
t.datetime "created_at", precision: nil, null: false t.datetime "created_at", precision: nil, null: false
t.integer "body_id", limit: 2, null: false t.integer "body_id", limit: 2, null: false
t.boolean "has_image", default: false, null: false t.boolean "has_image", default: false, null: false
@ -242,7 +242,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.datetime "reported_broken_at", precision: nil t.datetime "reported_broken_at", precision: nil
t.datetime "converted_at", precision: nil t.datetime "converted_at", precision: nil
t.boolean "image_manual", default: false, null: false t.boolean "image_manual", default: false, null: false
t.text "manifest", size: :medium t.text "manifest", size: :long
t.timestamp "manifest_cached_at" t.timestamp "manifest_cached_at"
t.string "known_glitches", limit: 128, default: "" t.string "known_glitches", limit: 128, default: ""
t.string "manifest_url" t.string "manifest_url"
@ -253,7 +253,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.index ["zone_id"], name: "idx_swf_assets_zone_id" t.index ["zone_id"], name: "idx_swf_assets_zone_id"
end end
create_table "users", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "users", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.string "name", limit: 20, null: false t.string "name", limit: 20, null: false
t.integer "auth_server_id", limit: 1, null: false t.integer "auth_server_id", limit: 1, null: false
t.integer "remote_id", null: false t.integer "remote_id", null: false
@ -268,7 +268,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_02_27_233743) do
t.boolean "support_staff", default: false, null: false t.boolean "support_staff", default: false, null: false
end end
create_table "zones", id: :integer, charset: "latin1", collation: "latin1_swedish_ci", force: :cascade do |t| create_table "zones", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_520_ci", force: :cascade do |t|
t.integer "depth" t.integer "depth"
t.integer "type_id" t.integer "type_id"
t.string "label", null: false t.string "label", null: false

12
deploy/setup.yml
View file

@ -384,6 +384,18 @@
skip-bind-address skip-bind-address
notify: Restart MariaDB notify: Restart MariaDB
# This is the best Unicode collation available in our version of MariaDB!
# We already specify it for all the tables in `schema.rb`, but also set it
# as the default collation for new tables here, too.
- name: Set MariaDb's default collation to utf8mb4_unicode_520_ci
copy:
dest: /etc/mysql/mariadb.conf.d/80-charsets.cnf
content: |
[mysqld]
character-set-server=utf8mb4
collation-server=utf8mb4_unicode_520_ci
notify: Restart MariaDB
- name: Enable slow query logging for MariaDB - name: Enable slow query logging for MariaDB
copy: copy:
dest: /etc/mysql/mariadb.conf.d/80-logging.cnf dest: /etc/mysql/mariadb.conf.d/80-logging.cnf

48
lib/tasks/db.rake Normal file
View file

@ -0,0 +1,48 @@
namespace :db do
desc "Generate SQL to convert all tables and columns to utf8mb4"
task :utf8_migration => :environment do
target_charset = "utf8mb4"
target_collation = "utf8mb4_unicode_520_ci"
db = ApplicationRecord.connection
# Don't mess with ActiveRecord's internal tables!
tables = db.tables.reject { |t| t.start_with? "ar_internal_" }
ups = []
downs = []
tables.sort.each do |table|
# If the table's default charset/collation doesn't match our goal, we'll
# modify it!
db.table_options(table) => {charset:, collation:}
if charset != target_charset || collation != target_collation
ups << "ALTER TABLE #{table} CONVERT TO CHARACTER SET " +
"#{target_charset} COLLATE #{target_collation}"
downs << "ALTER TABLE #{table} CONVERT TO CHARACTER SET " +
"#{charset} COLLATE #{collation}"
end
end
puts <<~MIGRATION_RB
reversible do |direction|
direction.up do
#{sql_as_ruby(ups).indent(2, "\t")}
end
direction.down do
#{sql_as_ruby(downs).indent(2, "\t")}
end
end
MIGRATION_RB
end
end
def sql_as_ruby(statements)
statements.map do |sql|
<<~EXECUTE_RB
execute <<-SQL
#{sql}
SQL
EXECUTE_RB
end.join("")
end