Migrate away from item translations for modeling

Note that my dev database wasn't in a great state to test
`yarn model-needed-items` directly—which is the only *actual* call
site for the modeling code, because `USE_NEW_MODELING=1` was never
turned on for the 2020 app I think?

But it's looking great on modeling directly when the server is run with
`USE_NEW_MODELING=1`, so I'm just gonna trust it for now, and check the
cron logs sometime.
This commit is contained in:
Emi Matchu 2024-02-20 16:49:26 -08:00
parent 8948b6567e
commit c3529450bf
3 changed files with 30 additions and 67 deletions

View file

@ -224,24 +224,6 @@ const buildItemLoader = (db) =>
);
});
const buildItemTranslationLoader = (db) =>
new DataLoader(async (itemIds) => {
const qs = itemIds.map((_) => "?").join(",");
const [rows] = await db.execute(
`SELECT * FROM item_translations WHERE item_id IN (${qs}) AND locale = "en"`,
itemIds,
);
const entities = rows.map(normalizeRow);
const entitiesByItemId = new Map(entities.map((e) => [e.itemId, e]));
return itemIds.map(
(itemId) =>
entitiesByItemId.get(String(itemId)) ||
new Error(`could not find translation for item ${itemId}`),
);
});
const buildItemByNameLoader = (db, loaders) =>
new DataLoader(
async (names) => {
@ -1470,7 +1452,6 @@ function buildLoaders(db) {
buildClosetHangersForDefaultListLoader(db);
loaders.colorLoader = buildColorLoader(db);
loaders.itemLoader = buildItemLoader(db);
loaders.itemTranslationLoader = buildItemTranslationLoader(db);
loaders.itemByNameLoader = buildItemByNameLoader(db, loaders);
loaders.itemSearchNumTotalItemsLoader =
buildItemSearchNumTotalItemsLoader(db);

View file

@ -33,7 +33,7 @@ async function saveModelingData(customPetData, petMetaData, context) {
const { db } = context;
await db.execute(
`INSERT INTO modeling_logs (log_json, pet_name) VALUES (?, ?)`,
[JSON.stringify(modelingLogs, null, 4), petMetaData.name]
[JSON.stringify(modelingLogs, null, 4), petMetaData.name],
);
}
}
@ -41,7 +41,7 @@ async function saveModelingData(customPetData, petMetaData, context) {
async function savePetTypeAndStateModelingData(
customPetData,
petMetaData,
context
context,
) {
// NOTE: When we automatically model items with "@imageHash" pet names, we
// can't load corresponding metadata. That's fine, the script is just looking
@ -133,7 +133,7 @@ async function savePetTypeAndStateModelingData(
biologyAssets.map((asset) => ({
type: "biology",
remoteId: String(asset.part_id),
}))
})),
),
]);
swfAssets = swfAssets.filter((sa) => sa != null);
@ -158,7 +158,7 @@ async function savePetTypeAndStateModelingData(
await db.execute(
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
VALUES ${qs};`,
values
values,
);
addToModelingLogs({
@ -172,46 +172,30 @@ async function savePetTypeAndStateModelingData(
}
async function saveItemModelingData(customPetData, context) {
const { db, itemLoader, itemTranslationLoader, addToModelingLogs } = context;
const { db, itemLoader, addToModelingLogs } = context;
const objectInfos = Object.values(customPetData.object_info_registry);
const incomingItems = objectInfos.map((objectInfo) => ({
id: String(objectInfo.obj_info_id),
name: objectInfo.name,
description: objectInfo.description,
zonesRestrict: objectInfo.zones_restrict,
thumbnailUrl: objectInfo.thumbnail_url,
category: objectInfo.category,
type: objectInfo.type,
rarity: objectInfo.rarity,
rarityIndex: objectInfo.rarity_index,
price: objectInfo.price,
weightLbs: objectInfo.weight_lbs,
}));
const incomingItemTranslations = objectInfos.map((objectInfo) => ({
itemId: String(objectInfo.obj_info_id),
locale: "en",
name: objectInfo.name,
description: objectInfo.description,
rarity: objectInfo.rarity,
}));
await Promise.all([
syncToDb(db, incomingItems, {
loader: itemLoader,
tableName: "items",
buildLoaderKey: (row) => row.id,
buildUpdateCondition: (row) => [`id = ?`, row.id],
addToModelingLogs,
}),
syncToDb(db, incomingItemTranslations, {
loader: itemTranslationLoader,
tableName: "item_translations",
buildLoaderKey: (row) => row.itemId,
buildUpdateCondition: (row) => [
`item_id = ? AND locale = "en"`,
row.itemId,
],
addToModelingLogs,
}),
]);
await syncToDb(db, incomingItems, {
loader: itemLoader,
tableName: "items",
buildLoaderKey: (row) => row.id,
buildUpdateCondition: (row) => [`id = ?`, row.id],
addToModelingLogs,
});
}
async function saveSwfAssetModelingData(customPetData, context) {
@ -313,7 +297,7 @@ async function saveSwfAssetModelingData(customPetData, context) {
// asset, despite only having remote_id available here. This saves
// us from another round-trip to SELECT the inserted IDs.
`(?, ?, ` +
`(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))`
`(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))`,
)
.join(", ");
const values = relationshipInserts
@ -327,7 +311,7 @@ async function saveSwfAssetModelingData(customPetData, context) {
await db.execute(
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
VALUES ${qs}`,
values
values,
);
addToModelingLogs({
@ -364,7 +348,7 @@ async function syncToDb(
includeUpdatedAt = true,
afterInsert = null,
addToModelingLogs,
}
},
) {
const loaderKeys = incomingRows.map(buildLoaderKey);
const currentRows = await loader.loadMany(loaderKeys);
@ -445,7 +429,7 @@ async function syncToDb(
// underscore-case instead of camel-case.
const rowKeys = Object.keys(inserts[0]).sort();
const columnNames = rowKeys.map((key) =>
key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase())
key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase()),
);
const columnsStr = columnNames.join(", ");
const qs = columnNames.map((_) => "?").join(", ");
@ -453,7 +437,7 @@ async function syncToDb(
const rowValues = inserts.map((row) => rowKeys.map((key) => row[key]));
await db.execute(
`INSERT INTO ${tableName} (${columnsStr}) VALUES ${rowQs};`,
rowValues.flat()
rowValues.flat(),
);
if (afterInsert) {
await afterInsert(inserts);
@ -469,15 +453,15 @@ async function syncToDb(
const rowKeys = Object.keys(update).sort();
const rowValues = rowKeys.map((k) => update[k]);
const columnNames = rowKeys.map((key) =>
key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase())
key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase()),
);
const qs = columnNames.map((c) => `${c} = ?`).join(", ");
const [conditionQs, ...conditionValues] = buildUpdateCondition(incomingRow);
updatePromises.push(
db.execute(
`UPDATE ${tableName} SET ${qs} WHERE ${conditionQs} LIMIT 1;`,
[...rowValues, ...conditionValues]
)
[...rowValues, ...conditionValues],
),
);
}
await Promise.all(updatePromises);

View file

@ -27,7 +27,7 @@ const resolvers = {
petAppearance: async (
{ name, customPetData, petMetaData },
_,
{ petTypeBySpeciesAndColorLoader, petStatesForPetTypeLoader }
{ petTypeBySpeciesAndColorLoader, petStatesForPetTypeLoader },
) => {
if (customPetData == null) {
return null;
@ -46,7 +46,7 @@ const resolvers = {
// First, look for a pet state containing exactly the same assets as this
// one.
const swfAssetIdsString = Object.values(
customPetData.custom_pet.biology_by_zone
customPetData.custom_pet.biology_by_zone,
)
.map((b) => b.part_id)
.sort((a, b) => Number(a) - Number(b))
@ -69,7 +69,7 @@ const resolvers = {
console.warn(
`Warning: For pet "${name}", fell back to pet state ${petState.id} ` +
`because it matches pose ${pose}. Actual pet state for these ` +
`assets not found: ${swfAssetIdsString}`
`assets not found: ${swfAssetIdsString}`,
);
return { id: petState.id };
}
@ -83,7 +83,7 @@ const resolvers = {
console.warn(
`Warning: For pet "${name}", fell back to pet state ${petState.id} ` +
`as an UNKNOWN fallback pose. Actual pet state for these ` +
`assets not found: ${swfAssetIdsString}`
`assets not found: ${swfAssetIdsString}`,
);
return { id: petState.id };
}
@ -93,7 +93,7 @@ const resolvers = {
// hasn't been saved yet.)
throw new Error(
`This pet's modeling data isn't loaded into our database yet, ` +
`sorry! Try using the Modeling Hub on Classic DTI to upload it first?`
`sorry! Try using the Modeling Hub on Classic DTI to upload it first?`,
);
},
wornItems: ({ customPetData }) => {
@ -119,9 +119,8 @@ const resolvers = {
petTypeBySpeciesAndColorLoader,
petStateByPetTypeAndAssetsLoader,
itemLoader,
itemTranslationLoader,
swfAssetByRemoteIdLoader,
}
},
) => {
const [customPetData, petMetaData] = await Promise.all([
loadCustomPetData(petName),
@ -141,7 +140,6 @@ const resolvers = {
petTypeBySpeciesAndColorLoader,
petStateByPetTypeAndAssetsLoader,
itemLoader,
itemTranslationLoader,
swfAssetByRemoteIdLoader,
});
}
@ -172,7 +170,7 @@ function getPoseFromPetData(petMetaData, petCustomData) {
throw new Error(
`could not identify pose: ` +
`moodId=${moodId}, ` +
`genderId=${genderId}`
`genderId=${genderId}`,
);
}
}