forked from OpenNeo/impress
Remove successfully-run item encoding repair script
This commit is contained in:
parent
83281591b3
commit
c241dc33b0
1 changed files with 0 additions and 150 deletions
|
|
@ -1,150 +0,0 @@
|
|||
namespace :db do
|
||||
desc "Fix double-encoded UTF-8 strings in item names and descriptions"
|
||||
task fix_double_encoding: :environment do
|
||||
puts "=" * 80
|
||||
puts "Fix Double-Encoded Strings in Database"
|
||||
puts "=" * 80
|
||||
puts
|
||||
|
||||
# Define the double-encoding patterns and their fixes
|
||||
# Each pattern maps: double-encoded string -> correct UTF-8 string
|
||||
# Using byte arrays to avoid encoding issues in the source file itself
|
||||
encoding_fixes = {
|
||||
# Common accented characters (Ă© => é, etc.)
|
||||
"\xC4\x82\xC2\xA9".force_encoding('UTF-8') => "\xC3\xA9".force_encoding('UTF-8'), # é
|
||||
"\xC4\x82\xC2\xB1".force_encoding('UTF-8') => "\xC3\xB1".force_encoding('UTF-8'), # ñ
|
||||
"\xC4\x82\xC2\xAD".force_encoding('UTF-8') => "\xC3\xAD".force_encoding('UTF-8'), # í
|
||||
"\xC4\x82\xC2\xA1".force_encoding('UTF-8') => "\xC3\xA1".force_encoding('UTF-8'), # á
|
||||
"\xC4\x82\xC2\xB3".force_encoding('UTF-8') => "\xC3\xB3".force_encoding('UTF-8'), # ó
|
||||
"\xC4\x82\xC2\xBA".force_encoding('UTF-8') => "\xC3\xBA".force_encoding('UTF-8'), # ú
|
||||
|
||||
# Smart quotes and apostrophes
|
||||
"\xC3\xA2\xE2\x82\xAC\xE2\x84\xA2".force_encoding('UTF-8') => "\xE2\x80\x99".force_encoding('UTF-8'), # '
|
||||
"\xC3\xA2\xE2\x82\xAC\xC5\x93".force_encoding('UTF-8') => "\xE2\x80\x9C".force_encoding('UTF-8'), # "
|
||||
"\xC3\xA2\xE2\x82\xAC\xC2\x9D".force_encoding('UTF-8') => "\xE2\x80\x9D".force_encoding('UTF-8'), # "
|
||||
"\xC3\xA2\xE2\x82\xAC\xCB\x9C".force_encoding('UTF-8') => "\xE2\x80\x98".force_encoding('UTF-8'), # '
|
||||
|
||||
# Other punctuation
|
||||
"\xC3\xA2\xE2\x82\xAC\xE2\x80\x9C".force_encoding('UTF-8') => "\xE2\x80\x93".force_encoding('UTF-8'), # –
|
||||
"\xC3\xA2\xE2\x82\xAC\xE2\x80\x9D".force_encoding('UTF-8') => "\xE2\x80\x94".force_encoding('UTF-8'), # —
|
||||
"\xC3\xA2\xE2\x82\xAC\xC2\xA6".force_encoding('UTF-8') => "\xE2\x80\xA6".force_encoding('UTF-8'), # …
|
||||
|
||||
# Non-breaking space
|
||||
"\xC3\x82\xC2\xA0".force_encoding('UTF-8') => "\xC2\xA0".force_encoding('UTF-8'),
|
||||
}
|
||||
|
||||
puts "Will fix the following patterns:"
|
||||
encoding_fixes.each do |bad, good|
|
||||
puts " #{bad.inspect} → #{good.inspect}"
|
||||
end
|
||||
puts
|
||||
|
||||
# Find affected items by actually checking for the pattern in Ruby
|
||||
# (MySQL LIKE queries give false positives with multi-byte UTF-8)
|
||||
puts "Scanning items for double-encoding patterns..."
|
||||
|
||||
items_by_pattern = {}
|
||||
total_affected = Set.new
|
||||
count_by_pattern = Hash.new { |h, k| h[k] = { name: 0, description: 0 } }
|
||||
|
||||
Item.find_each do |item|
|
||||
encoding_fixes.each_key do |pattern|
|
||||
if item.name.include?(pattern)
|
||||
items_by_pattern[pattern] ||= { name: [], description: [] }
|
||||
items_by_pattern[pattern][:name] << item.id
|
||||
total_affected << item.id
|
||||
count_by_pattern[pattern][:name] += 1
|
||||
end
|
||||
|
||||
if item.description.include?(pattern)
|
||||
items_by_pattern[pattern] ||= { name: [], description: [] }
|
||||
items_by_pattern[pattern][:description] << item.id
|
||||
total_affected << item.id
|
||||
count_by_pattern[pattern][:description] += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
puts
|
||||
count_by_pattern.each do |pattern, counts|
|
||||
puts "#{pattern.inspect}: #{counts[:name]} names, #{counts[:description]} descriptions"
|
||||
end
|
||||
|
||||
puts
|
||||
puts "Total affected items: #{total_affected.size}"
|
||||
puts
|
||||
|
||||
if total_affected.empty?
|
||||
puts "No items need fixing!"
|
||||
next
|
||||
end
|
||||
|
||||
# Show some examples
|
||||
puts "Example affected items:"
|
||||
Item.where(id: total_affected.to_a.first(5)).each do |item|
|
||||
puts " #{item.id}: #{item.name}"
|
||||
end
|
||||
puts " ... and #{total_affected.size - 5} more" if total_affected.size > 5
|
||||
puts
|
||||
|
||||
# Ask for confirmation
|
||||
print "Fix these items by replacing double-encoded characters? (y/N): "
|
||||
response = STDIN.gets.chomp
|
||||
unless response.downcase == 'y'
|
||||
puts "Aborted."
|
||||
next
|
||||
end
|
||||
|
||||
puts
|
||||
puts "Fixing items..."
|
||||
puts "-" * 80
|
||||
|
||||
fixed_count = 0
|
||||
no_change_count = 0
|
||||
|
||||
Item.where(id: total_affected.to_a).find_each do |item|
|
||||
original_name = item.name
|
||||
original_description = item.description
|
||||
|
||||
# Apply all fixes to name and description
|
||||
new_name = original_name.dup
|
||||
new_description = original_description.dup
|
||||
|
||||
encoding_fixes.each do |bad, good|
|
||||
new_name.gsub!(bad, good)
|
||||
new_description.gsub!(bad, good)
|
||||
end
|
||||
|
||||
# Only save if something changed
|
||||
if new_name != original_name || new_description != original_description
|
||||
item.name = new_name
|
||||
item.description = new_description
|
||||
item.save!(validate: false) # Skip validations to avoid potential issues
|
||||
|
||||
if new_name != original_name
|
||||
puts "#{item.id}: #{original_name.inspect} → #{new_name.inspect}"
|
||||
elsif new_description != original_description
|
||||
puts "#{item.id}: Updated description only"
|
||||
end
|
||||
|
||||
fixed_count += 1
|
||||
else
|
||||
no_change_count += 1
|
||||
end
|
||||
end
|
||||
|
||||
puts
|
||||
puts "-" * 80
|
||||
puts "Complete!"
|
||||
puts " ✓ Fixed: #{fixed_count}"
|
||||
puts " ⊘ No changes needed: #{no_change_count}"
|
||||
puts
|
||||
|
||||
# Show a sample of fixed items
|
||||
puts "Sample of fixed items:"
|
||||
Item.where(id: total_affected.to_a.first(5)).each do |item|
|
||||
puts " #{item.id}: #{item.name}"
|
||||
end
|
||||
puts
|
||||
end
|
||||
end
|
||||
Loading…
Reference in a new issue