OpenNeo/impress-2020
OpenNeo/impress-2020
1
0
Fork 0
Code Pull requests Activity

add logging to modeling

Browse source 
my hope is that, if we fuck things up, this will make it clear 😅
This commit is contained in:
Emi Matchu 2020-10-06 07:06:19 -07:00
parent bb812a2b81
commit 99e6480486
5 changed files with 97 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -45,7 +45,7 @@
"mysql-dev": "mysql --host=localhost --user=impress_2020_dev --password=impress_2020_dev --database=impress_2020_dev",
"mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress",
"mysqldump": "mysqldump --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --column-statistics=0",
"download-mysql-schema": "yarn --silent mysqldump --no-data openneo_impress items item_translations parents_swf_assets pet_types pet_states swf_assets | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema.sql && yarn --silent mysqldump openneo_impress species species_translations colors color_translations zones zone_translations > scripts/setup-mysql-dev-constants.sql",
"download-mysql-schema": "yarn --silent mysqldump --no-data openneo_impress items item_translations modeling_logs parents_swf_assets pet_types pet_states swf_assets | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema.sql && yarn --silent mysqldump openneo_impress species species_translations colors color_translations zones zone_translations > scripts/setup-mysql-dev-constants.sql",
"setup-mysql": "yarn mysql-admin < scripts/setup-mysql.sql",
"setup-mysql-dev": "yarn mysql-dev < scripts/setup-mysql-dev-constants.sql && yarn mysql-dev < scripts/setup-mysql-dev-schema.sql",
"build-cached-data": "node -r dotenv/config scripts/build-cached-data.js",

2
scripts/setup-mysql-dev-constants.sql
View file

@ -189,4 +189,4 @@ UNLOCK TABLES;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2020-10-06 6:09:54
-- Dump completed on 2020-10-06 6:59:56

18
scripts/setup-mysql-dev-schema.sql
View file

@ -68,6 +68,22 @@ CREATE TABLE `item_translations` (
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `modeling_logs`
--
DROP TABLE IF EXISTS `modeling_logs`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `modeling_logs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`log_json` text NOT NULL,
`pet_name` varchar(128) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `parents_swf_assets`
--
@ -167,4 +183,4 @@ CREATE TABLE `swf_assets` (
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2020-10-06 6:09:45
-- Dump completed on 2020-10-06 6:59:49

14
scripts/setup-mysql.sql
View file

@ -5,6 +5,7 @@ GRANT SELECT ON colors TO impress2020;
GRANT SELECT ON color_translations TO impress2020;
GRANT SELECT ON items TO impress2020;
GRANT SELECT ON item_translations TO impress2020;
GRANT SELECT ON modeling_logs TO impress2020;
GRANT SELECT ON parents_swf_assets TO impress2020;
GRANT SELECT ON pet_types TO impress2020;
GRANT SELECT ON pet_states TO impress2020;
@ -14,11 +15,14 @@ GRANT SELECT ON swf_assets TO impress2020;
GRANT SELECT ON zones TO impress2020;
GRANT SELECT ON zone_translations TO impress2020;
-- Public data tables: write
GRANT UPDATE ON items TO impress2020;
GRANT DELETE ON parents_swf_assets TO impress2020;
GRANT UPDATE ON pet_states TO impress2020;
GRANT UPDATE ON swf_assets TO impress2020;
-- Public data tables: write. Used in modeling and support tools.
GRANT INSERT, UPDATE ON items TO impress2020;
GRANT INSERT, UPDATE ON item_translations TO impress2020;
GRANT INSERT, UPDATE, DELETE ON parents_swf_assets TO impress2020;
GRANT INSERT, UPDATE ON pet_types TO impress2020;
GRANT INSERT, UPDATE ON pet_states TO impress2020;
GRANT INSERT, UPDATE ON swf_assets TO impress2020;
GRANT INSERT ON modeling_logs TO impress2020;
-- User data tables
GRANT SELECT ON closet_hangers TO impress2020;

78
src/server/modeling.js
View file

@ -8,11 +8,26 @@
* pet contains data we haven't seen before, we write!
*/
async function saveModelingData(customPetData, petMetaData, context) {
const modelingLogs = [];
const addToModelingLogs = (entry) => {
console.log("[Modeling] " + JSON.stringify(entry, null, 4));
modelingLogs.push(entry);
};
context = { ...context, addToModelingLogs };
await Promise.all([
savePetTypeAndStateModelingData(customPetData, petMetaData, context),
saveItemModelingData(customPetData, context),
saveSwfAssetModelingData(customPetData, context),
]);
if (modelingLogs.length > 0) {
const { db } = context;
await db.execute(
`INSERT INTO modeling_logs (log_json, pet_name) VALUES (?, ?)`,
[JSON.stringify(modelingLogs, null, 4), petMetaData.name]
);
}
}
async function savePetTypeAndStateModelingData(
@ -25,7 +40,9 @@ async function savePetTypeAndStateModelingData(
petTypeBySpeciesAndColorLoader,
petStateByPetTypeAndAssetsLoader,
swfAssetByRemoteIdLoader,
addToModelingLogs,
} = context;
const incomingPetType = {
colorId: String(customPetData.custom_pet.color_id),
speciesId: String(customPetData.custom_pet.species_id),
@ -50,6 +67,7 @@ async function savePetTypeAndStateModelingData(
row.colorId,
],
includeUpdatedAt: false,
addToModelingLogs,
});
// NOTE: This pet type should have been looked up when syncing pet type, so
@ -107,21 +125,39 @@ async function savePetTypeAndStateModelingData(
if (swfAssets.length === 0) {
throw new Error(`pet state ${petState.id} has no saved assets?`);
}
const relationshipInserts = swfAssets.map((sa) => ({
parentType: "PetState",
parentId: petState.id,
swfAssetId: sa.id,
}));
const qs = swfAssets.map((_) => `(?, ?, ?)`).join(", ");
const values = swfAssets
.map((sa) => ["PetState", petState.id, sa.id])
const values = relationshipInserts
.map(({ parentType, parentId, swfAssetId }) => [
parentType,
parentId,
swfAssetId,
])
.flat();
await db.execute(
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
VALUES ${qs};`,
values
);
addToModelingLogs({
tableName: "parents_swf_assets",
inserts: relationshipInserts,
updates: [],
});
},
addToModelingLogs,
});
}
async function saveItemModelingData(customPetData, context) {
const { db, itemLoader, itemTranslationLoader } = context;
const { db, itemLoader, itemTranslationLoader, addToModelingLogs } = context;
const objectInfos = Object.values(customPetData.object_info_registry);
const incomingItems = objectInfos.map((objectInfo) => ({
@ -148,6 +184,7 @@ async function saveItemModelingData(customPetData, context) {
tableName: "items",
buildLoaderKey: (row) => row.id,
buildUpdateCondition: (row) => [`id = ?`, row.id],
addToModelingLogs,
}),
syncToDb(db, incomingItemTranslations, {
loader: itemTranslationLoader,
@ -157,12 +194,13 @@ async function saveItemModelingData(customPetData, context) {
`item_id = ? AND locale = "en"`,
row.itemId,
],
addToModelingLogs,
}),
]);
}
async function saveSwfAssetModelingData(customPetData, context) {
const { db, swfAssetByRemoteIdLoader } = context;
const { db, swfAssetByRemoteIdLoader, addToModelingLogs } = context;
const objectAssets = Object.values(customPetData.object_asset_registry);
const incomingItemSwfAssets = objectAssets.map((objectAsset) => ({
@ -244,6 +282,12 @@ async function saveSwfAssetModelingData(customPetData, context) {
return;
}
const relationshipInserts = itemAssetInserts.map(({ remoteId }) => ({
parentType: "Item",
parentId: assetIdToItemIdMap.get(remoteId),
remoteId,
}));
const qs = itemAssetInserts
.map(
(_) =>
@ -254,11 +298,11 @@ async function saveSwfAssetModelingData(customPetData, context) {
`(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))`
)
.join(", ");
const values = itemAssetInserts
.map(({ remoteId: swfAssetId }) => [
"Item",
assetIdToItemIdMap.get(swfAssetId),
swfAssetId,
const values = relationshipInserts
.map(({ parentType, parentId, remoteId }) => [
parentType,
parentId,
remoteId,
])
.flat();
@ -267,7 +311,14 @@ async function saveSwfAssetModelingData(customPetData, context) {
VALUES ${qs}`,
values
);
addToModelingLogs({
tableName: "parents_swf_assets",
inserts: relationshipInserts,
updates: [],
});
},
addToModelingLogs,
});
}
@ -294,6 +345,7 @@ async function syncToDb(
includeCreatedAt = true,
includeUpdatedAt = true,
afterInsert = null,
addToModelingLogs,
}
) {
const loaderKeys = incomingRows.map(buildLoaderKey);
@ -411,6 +463,14 @@ async function syncToDb(
);
}
await Promise.all(updatePromises);
if (inserts.length > 0 || updates.length > 0) {
addToModelingLogs({
tableName,
inserts,
updates,
});
}
}
module.exports = { saveModelingData };