Migrate away from item translations for modeling

Note that my dev database wasn't in a great state to test
`yarn model-needed-items` directly—which is the only *actual* call
site for the modeling code, because `USE_NEW_MODELING=1` was never
turned on for the 2020 app I think?

But it's looking great on modeling directly when the server is run with
`USE_NEW_MODELING=1`, so I'm just gonna trust it for now, and check the
cron logs sometime.
This commit is contained in:
Emi Matchu 2024-02-20 16:49:26 -08:00
parent 8948b6567e
commit c3529450bf
3 changed files with 30 additions and 67 deletions

View file

@ -224,24 +224,6 @@ const buildItemLoader = (db) =>
); );
}); });
const buildItemTranslationLoader = (db) =>
new DataLoader(async (itemIds) => {
const qs = itemIds.map((_) => "?").join(",");
const [rows] = await db.execute(
`SELECT * FROM item_translations WHERE item_id IN (${qs}) AND locale = "en"`,
itemIds,
);
const entities = rows.map(normalizeRow);
const entitiesByItemId = new Map(entities.map((e) => [e.itemId, e]));
return itemIds.map(
(itemId) =>
entitiesByItemId.get(String(itemId)) ||
new Error(`could not find translation for item ${itemId}`),
);
});
const buildItemByNameLoader = (db, loaders) => const buildItemByNameLoader = (db, loaders) =>
new DataLoader( new DataLoader(
async (names) => { async (names) => {
@ -1470,7 +1452,6 @@ function buildLoaders(db) {
buildClosetHangersForDefaultListLoader(db); buildClosetHangersForDefaultListLoader(db);
loaders.colorLoader = buildColorLoader(db); loaders.colorLoader = buildColorLoader(db);
loaders.itemLoader = buildItemLoader(db); loaders.itemLoader = buildItemLoader(db);
loaders.itemTranslationLoader = buildItemTranslationLoader(db);
loaders.itemByNameLoader = buildItemByNameLoader(db, loaders); loaders.itemByNameLoader = buildItemByNameLoader(db, loaders);
loaders.itemSearchNumTotalItemsLoader = loaders.itemSearchNumTotalItemsLoader =
buildItemSearchNumTotalItemsLoader(db); buildItemSearchNumTotalItemsLoader(db);

View file

@ -33,7 +33,7 @@ async function saveModelingData(customPetData, petMetaData, context) {
const { db } = context; const { db } = context;
await db.execute( await db.execute(
`INSERT INTO modeling_logs (log_json, pet_name) VALUES (?, ?)`, `INSERT INTO modeling_logs (log_json, pet_name) VALUES (?, ?)`,
[JSON.stringify(modelingLogs, null, 4), petMetaData.name] [JSON.stringify(modelingLogs, null, 4), petMetaData.name],
); );
} }
} }
@ -41,7 +41,7 @@ async function saveModelingData(customPetData, petMetaData, context) {
async function savePetTypeAndStateModelingData( async function savePetTypeAndStateModelingData(
customPetData, customPetData,
petMetaData, petMetaData,
context context,
) { ) {
// NOTE: When we automatically model items with "@imageHash" pet names, we // NOTE: When we automatically model items with "@imageHash" pet names, we
// can't load corresponding metadata. That's fine, the script is just looking // can't load corresponding metadata. That's fine, the script is just looking
@ -133,7 +133,7 @@ async function savePetTypeAndStateModelingData(
biologyAssets.map((asset) => ({ biologyAssets.map((asset) => ({
type: "biology", type: "biology",
remoteId: String(asset.part_id), remoteId: String(asset.part_id),
})) })),
), ),
]); ]);
swfAssets = swfAssets.filter((sa) => sa != null); swfAssets = swfAssets.filter((sa) => sa != null);
@ -158,7 +158,7 @@ async function savePetTypeAndStateModelingData(
await db.execute( await db.execute(
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id) `INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
VALUES ${qs};`, VALUES ${qs};`,
values values,
); );
addToModelingLogs({ addToModelingLogs({
@ -172,46 +172,30 @@ async function savePetTypeAndStateModelingData(
} }
async function saveItemModelingData(customPetData, context) { async function saveItemModelingData(customPetData, context) {
const { db, itemLoader, itemTranslationLoader, addToModelingLogs } = context; const { db, itemLoader, addToModelingLogs } = context;
const objectInfos = Object.values(customPetData.object_info_registry); const objectInfos = Object.values(customPetData.object_info_registry);
const incomingItems = objectInfos.map((objectInfo) => ({ const incomingItems = objectInfos.map((objectInfo) => ({
id: String(objectInfo.obj_info_id), id: String(objectInfo.obj_info_id),
name: objectInfo.name,
description: objectInfo.description,
zonesRestrict: objectInfo.zones_restrict, zonesRestrict: objectInfo.zones_restrict,
thumbnailUrl: objectInfo.thumbnail_url, thumbnailUrl: objectInfo.thumbnail_url,
category: objectInfo.category, category: objectInfo.category,
type: objectInfo.type, type: objectInfo.type,
rarity: objectInfo.rarity,
rarityIndex: objectInfo.rarity_index, rarityIndex: objectInfo.rarity_index,
price: objectInfo.price, price: objectInfo.price,
weightLbs: objectInfo.weight_lbs, weightLbs: objectInfo.weight_lbs,
})); }));
const incomingItemTranslations = objectInfos.map((objectInfo) => ({
itemId: String(objectInfo.obj_info_id),
locale: "en",
name: objectInfo.name,
description: objectInfo.description,
rarity: objectInfo.rarity,
}));
await Promise.all([ await syncToDb(db, incomingItems, {
syncToDb(db, incomingItems, { loader: itemLoader,
loader: itemLoader, tableName: "items",
tableName: "items", buildLoaderKey: (row) => row.id,
buildLoaderKey: (row) => row.id, buildUpdateCondition: (row) => [`id = ?`, row.id],
buildUpdateCondition: (row) => [`id = ?`, row.id], addToModelingLogs,
addToModelingLogs, });
}),
syncToDb(db, incomingItemTranslations, {
loader: itemTranslationLoader,
tableName: "item_translations",
buildLoaderKey: (row) => row.itemId,
buildUpdateCondition: (row) => [
`item_id = ? AND locale = "en"`,
row.itemId,
],
addToModelingLogs,
}),
]);
} }
async function saveSwfAssetModelingData(customPetData, context) { async function saveSwfAssetModelingData(customPetData, context) {
@ -313,7 +297,7 @@ async function saveSwfAssetModelingData(customPetData, context) {
// asset, despite only having remote_id available here. This saves // asset, despite only having remote_id available here. This saves
// us from another round-trip to SELECT the inserted IDs. // us from another round-trip to SELECT the inserted IDs.
`(?, ?, ` + `(?, ?, ` +
`(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))` `(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))`,
) )
.join(", "); .join(", ");
const values = relationshipInserts const values = relationshipInserts
@ -327,7 +311,7 @@ async function saveSwfAssetModelingData(customPetData, context) {
await db.execute( await db.execute(
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id) `INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
VALUES ${qs}`, VALUES ${qs}`,
values values,
); );
addToModelingLogs({ addToModelingLogs({
@ -364,7 +348,7 @@ async function syncToDb(
includeUpdatedAt = true, includeUpdatedAt = true,
afterInsert = null, afterInsert = null,
addToModelingLogs, addToModelingLogs,
} },
) { ) {
const loaderKeys = incomingRows.map(buildLoaderKey); const loaderKeys = incomingRows.map(buildLoaderKey);
const currentRows = await loader.loadMany(loaderKeys); const currentRows = await loader.loadMany(loaderKeys);
@ -445,7 +429,7 @@ async function syncToDb(
// underscore-case instead of camel-case. // underscore-case instead of camel-case.
const rowKeys = Object.keys(inserts[0]).sort(); const rowKeys = Object.keys(inserts[0]).sort();
const columnNames = rowKeys.map((key) => const columnNames = rowKeys.map((key) =>
key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase()) key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase()),
); );
const columnsStr = columnNames.join(", "); const columnsStr = columnNames.join(", ");
const qs = columnNames.map((_) => "?").join(", "); const qs = columnNames.map((_) => "?").join(", ");
@ -453,7 +437,7 @@ async function syncToDb(
const rowValues = inserts.map((row) => rowKeys.map((key) => row[key])); const rowValues = inserts.map((row) => rowKeys.map((key) => row[key]));
await db.execute( await db.execute(
`INSERT INTO ${tableName} (${columnsStr}) VALUES ${rowQs};`, `INSERT INTO ${tableName} (${columnsStr}) VALUES ${rowQs};`,
rowValues.flat() rowValues.flat(),
); );
if (afterInsert) { if (afterInsert) {
await afterInsert(inserts); await afterInsert(inserts);
@ -469,15 +453,15 @@ async function syncToDb(
const rowKeys = Object.keys(update).sort(); const rowKeys = Object.keys(update).sort();
const rowValues = rowKeys.map((k) => update[k]); const rowValues = rowKeys.map((k) => update[k]);
const columnNames = rowKeys.map((key) => const columnNames = rowKeys.map((key) =>
key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase()) key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase()),
); );
const qs = columnNames.map((c) => `${c} = ?`).join(", "); const qs = columnNames.map((c) => `${c} = ?`).join(", ");
const [conditionQs, ...conditionValues] = buildUpdateCondition(incomingRow); const [conditionQs, ...conditionValues] = buildUpdateCondition(incomingRow);
updatePromises.push( updatePromises.push(
db.execute( db.execute(
`UPDATE ${tableName} SET ${qs} WHERE ${conditionQs} LIMIT 1;`, `UPDATE ${tableName} SET ${qs} WHERE ${conditionQs} LIMIT 1;`,
[...rowValues, ...conditionValues] [...rowValues, ...conditionValues],
) ),
); );
} }
await Promise.all(updatePromises); await Promise.all(updatePromises);

View file

@ -27,7 +27,7 @@ const resolvers = {
petAppearance: async ( petAppearance: async (
{ name, customPetData, petMetaData }, { name, customPetData, petMetaData },
_, _,
{ petTypeBySpeciesAndColorLoader, petStatesForPetTypeLoader } { petTypeBySpeciesAndColorLoader, petStatesForPetTypeLoader },
) => { ) => {
if (customPetData == null) { if (customPetData == null) {
return null; return null;
@ -46,7 +46,7 @@ const resolvers = {
// First, look for a pet state containing exactly the same assets as this // First, look for a pet state containing exactly the same assets as this
// one. // one.
const swfAssetIdsString = Object.values( const swfAssetIdsString = Object.values(
customPetData.custom_pet.biology_by_zone customPetData.custom_pet.biology_by_zone,
) )
.map((b) => b.part_id) .map((b) => b.part_id)
.sort((a, b) => Number(a) - Number(b)) .sort((a, b) => Number(a) - Number(b))
@ -69,7 +69,7 @@ const resolvers = {
console.warn( console.warn(
`Warning: For pet "${name}", fell back to pet state ${petState.id} ` + `Warning: For pet "${name}", fell back to pet state ${petState.id} ` +
`because it matches pose ${pose}. Actual pet state for these ` + `because it matches pose ${pose}. Actual pet state for these ` +
`assets not found: ${swfAssetIdsString}` `assets not found: ${swfAssetIdsString}`,
); );
return { id: petState.id }; return { id: petState.id };
} }
@ -83,7 +83,7 @@ const resolvers = {
console.warn( console.warn(
`Warning: For pet "${name}", fell back to pet state ${petState.id} ` + `Warning: For pet "${name}", fell back to pet state ${petState.id} ` +
`as an UNKNOWN fallback pose. Actual pet state for these ` + `as an UNKNOWN fallback pose. Actual pet state for these ` +
`assets not found: ${swfAssetIdsString}` `assets not found: ${swfAssetIdsString}`,
); );
return { id: petState.id }; return { id: petState.id };
} }
@ -93,7 +93,7 @@ const resolvers = {
// hasn't been saved yet.) // hasn't been saved yet.)
throw new Error( throw new Error(
`This pet's modeling data isn't loaded into our database yet, ` + `This pet's modeling data isn't loaded into our database yet, ` +
`sorry! Try using the Modeling Hub on Classic DTI to upload it first?` `sorry! Try using the Modeling Hub on Classic DTI to upload it first?`,
); );
}, },
wornItems: ({ customPetData }) => { wornItems: ({ customPetData }) => {
@ -119,9 +119,8 @@ const resolvers = {
petTypeBySpeciesAndColorLoader, petTypeBySpeciesAndColorLoader,
petStateByPetTypeAndAssetsLoader, petStateByPetTypeAndAssetsLoader,
itemLoader, itemLoader,
itemTranslationLoader,
swfAssetByRemoteIdLoader, swfAssetByRemoteIdLoader,
} },
) => { ) => {
const [customPetData, petMetaData] = await Promise.all([ const [customPetData, petMetaData] = await Promise.all([
loadCustomPetData(petName), loadCustomPetData(petName),
@ -141,7 +140,6 @@ const resolvers = {
petTypeBySpeciesAndColorLoader, petTypeBySpeciesAndColorLoader,
petStateByPetTypeAndAssetsLoader, petStateByPetTypeAndAssetsLoader,
itemLoader, itemLoader,
itemTranslationLoader,
swfAssetByRemoteIdLoader, swfAssetByRemoteIdLoader,
}); });
} }
@ -172,7 +170,7 @@ function getPoseFromPetData(petMetaData, petCustomData) {
throw new Error( throw new Error(
`could not identify pose: ` + `could not identify pose: ` +
`moodId=${moodId}, ` + `moodId=${moodId}, ` +
`genderId=${genderId}` `genderId=${genderId}`,
); );
} }
} }