add logging to modeling
my hope is that, if we fuck things up, this will make it clear 😅
This commit is contained in:
parent
bb812a2b81
commit
99e6480486
5 changed files with 97 additions and 17 deletions
|
@ -45,7 +45,7 @@
|
||||||
"mysql-dev": "mysql --host=localhost --user=impress_2020_dev --password=impress_2020_dev --database=impress_2020_dev",
|
"mysql-dev": "mysql --host=localhost --user=impress_2020_dev --password=impress_2020_dev --database=impress_2020_dev",
|
||||||
"mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress",
|
"mysql-admin": "mysql --host=impress.openneo.net --user=matchu --password --database=openneo_impress",
|
||||||
"mysqldump": "mysqldump --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --column-statistics=0",
|
"mysqldump": "mysqldump --host=impress.openneo.net --user=$(dotenv -p IMPRESS_MYSQL_USER) --password=$(dotenv -p IMPRESS_MYSQL_PASSWORD) --column-statistics=0",
|
||||||
"download-mysql-schema": "yarn --silent mysqldump --no-data openneo_impress items item_translations parents_swf_assets pet_types pet_states swf_assets | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema.sql && yarn --silent mysqldump openneo_impress species species_translations colors color_translations zones zone_translations > scripts/setup-mysql-dev-constants.sql",
|
"download-mysql-schema": "yarn --silent mysqldump --no-data openneo_impress items item_translations modeling_logs parents_swf_assets pet_types pet_states swf_assets | sed 's/ AUTO_INCREMENT=[0-9]*//g' > scripts/setup-mysql-dev-schema.sql && yarn --silent mysqldump openneo_impress species species_translations colors color_translations zones zone_translations > scripts/setup-mysql-dev-constants.sql",
|
||||||
"setup-mysql": "yarn mysql-admin < scripts/setup-mysql.sql",
|
"setup-mysql": "yarn mysql-admin < scripts/setup-mysql.sql",
|
||||||
"setup-mysql-dev": "yarn mysql-dev < scripts/setup-mysql-dev-constants.sql && yarn mysql-dev < scripts/setup-mysql-dev-schema.sql",
|
"setup-mysql-dev": "yarn mysql-dev < scripts/setup-mysql-dev-constants.sql && yarn mysql-dev < scripts/setup-mysql-dev-schema.sql",
|
||||||
"build-cached-data": "node -r dotenv/config scripts/build-cached-data.js",
|
"build-cached-data": "node -r dotenv/config scripts/build-cached-data.js",
|
||||||
|
|
|
@ -189,4 +189,4 @@ UNLOCK TABLES;
|
||||||
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
|
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
|
||||||
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
||||||
|
|
||||||
-- Dump completed on 2020-10-06 6:09:54
|
-- Dump completed on 2020-10-06 6:59:56
|
||||||
|
|
|
@ -68,6 +68,22 @@ CREATE TABLE `item_translations` (
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
|
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
|
||||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||||
|
|
||||||
|
--
|
||||||
|
-- Table structure for table `modeling_logs`
|
||||||
|
--
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS `modeling_logs`;
|
||||||
|
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
||||||
|
/*!50503 SET character_set_client = utf8mb4 */;
|
||||||
|
CREATE TABLE `modeling_logs` (
|
||||||
|
`id` int(11) NOT NULL AUTO_INCREMENT,
|
||||||
|
`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
`log_json` text NOT NULL,
|
||||||
|
`pet_name` varchar(128) NOT NULL,
|
||||||
|
PRIMARY KEY (`id`)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
|
||||||
|
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||||
|
|
||||||
--
|
--
|
||||||
-- Table structure for table `parents_swf_assets`
|
-- Table structure for table `parents_swf_assets`
|
||||||
--
|
--
|
||||||
|
@ -167,4 +183,4 @@ CREATE TABLE `swf_assets` (
|
||||||
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
|
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
|
||||||
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
||||||
|
|
||||||
-- Dump completed on 2020-10-06 6:09:45
|
-- Dump completed on 2020-10-06 6:59:49
|
||||||
|
|
|
@ -5,6 +5,7 @@ GRANT SELECT ON colors TO impress2020;
|
||||||
GRANT SELECT ON color_translations TO impress2020;
|
GRANT SELECT ON color_translations TO impress2020;
|
||||||
GRANT SELECT ON items TO impress2020;
|
GRANT SELECT ON items TO impress2020;
|
||||||
GRANT SELECT ON item_translations TO impress2020;
|
GRANT SELECT ON item_translations TO impress2020;
|
||||||
|
GRANT SELECT ON modeling_logs TO impress2020;
|
||||||
GRANT SELECT ON parents_swf_assets TO impress2020;
|
GRANT SELECT ON parents_swf_assets TO impress2020;
|
||||||
GRANT SELECT ON pet_types TO impress2020;
|
GRANT SELECT ON pet_types TO impress2020;
|
||||||
GRANT SELECT ON pet_states TO impress2020;
|
GRANT SELECT ON pet_states TO impress2020;
|
||||||
|
@ -14,11 +15,14 @@ GRANT SELECT ON swf_assets TO impress2020;
|
||||||
GRANT SELECT ON zones TO impress2020;
|
GRANT SELECT ON zones TO impress2020;
|
||||||
GRANT SELECT ON zone_translations TO impress2020;
|
GRANT SELECT ON zone_translations TO impress2020;
|
||||||
|
|
||||||
-- Public data tables: write
|
-- Public data tables: write. Used in modeling and support tools.
|
||||||
GRANT UPDATE ON items TO impress2020;
|
GRANT INSERT, UPDATE ON items TO impress2020;
|
||||||
GRANT DELETE ON parents_swf_assets TO impress2020;
|
GRANT INSERT, UPDATE ON item_translations TO impress2020;
|
||||||
GRANT UPDATE ON pet_states TO impress2020;
|
GRANT INSERT, UPDATE, DELETE ON parents_swf_assets TO impress2020;
|
||||||
GRANT UPDATE ON swf_assets TO impress2020;
|
GRANT INSERT, UPDATE ON pet_types TO impress2020;
|
||||||
|
GRANT INSERT, UPDATE ON pet_states TO impress2020;
|
||||||
|
GRANT INSERT, UPDATE ON swf_assets TO impress2020;
|
||||||
|
GRANT INSERT ON modeling_logs TO impress2020;
|
||||||
|
|
||||||
-- User data tables
|
-- User data tables
|
||||||
GRANT SELECT ON closet_hangers TO impress2020;
|
GRANT SELECT ON closet_hangers TO impress2020;
|
||||||
|
|
|
@ -8,11 +8,26 @@
|
||||||
* pet contains data we haven't seen before, we write!
|
* pet contains data we haven't seen before, we write!
|
||||||
*/
|
*/
|
||||||
async function saveModelingData(customPetData, petMetaData, context) {
|
async function saveModelingData(customPetData, petMetaData, context) {
|
||||||
|
const modelingLogs = [];
|
||||||
|
const addToModelingLogs = (entry) => {
|
||||||
|
console.log("[Modeling] " + JSON.stringify(entry, null, 4));
|
||||||
|
modelingLogs.push(entry);
|
||||||
|
};
|
||||||
|
context = { ...context, addToModelingLogs };
|
||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
savePetTypeAndStateModelingData(customPetData, petMetaData, context),
|
savePetTypeAndStateModelingData(customPetData, petMetaData, context),
|
||||||
saveItemModelingData(customPetData, context),
|
saveItemModelingData(customPetData, context),
|
||||||
saveSwfAssetModelingData(customPetData, context),
|
saveSwfAssetModelingData(customPetData, context),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
if (modelingLogs.length > 0) {
|
||||||
|
const { db } = context;
|
||||||
|
await db.execute(
|
||||||
|
`INSERT INTO modeling_logs (log_json, pet_name) VALUES (?, ?)`,
|
||||||
|
[JSON.stringify(modelingLogs, null, 4), petMetaData.name]
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function savePetTypeAndStateModelingData(
|
async function savePetTypeAndStateModelingData(
|
||||||
|
@ -25,7 +40,9 @@ async function savePetTypeAndStateModelingData(
|
||||||
petTypeBySpeciesAndColorLoader,
|
petTypeBySpeciesAndColorLoader,
|
||||||
petStateByPetTypeAndAssetsLoader,
|
petStateByPetTypeAndAssetsLoader,
|
||||||
swfAssetByRemoteIdLoader,
|
swfAssetByRemoteIdLoader,
|
||||||
|
addToModelingLogs,
|
||||||
} = context;
|
} = context;
|
||||||
|
|
||||||
const incomingPetType = {
|
const incomingPetType = {
|
||||||
colorId: String(customPetData.custom_pet.color_id),
|
colorId: String(customPetData.custom_pet.color_id),
|
||||||
speciesId: String(customPetData.custom_pet.species_id),
|
speciesId: String(customPetData.custom_pet.species_id),
|
||||||
|
@ -50,6 +67,7 @@ async function savePetTypeAndStateModelingData(
|
||||||
row.colorId,
|
row.colorId,
|
||||||
],
|
],
|
||||||
includeUpdatedAt: false,
|
includeUpdatedAt: false,
|
||||||
|
addToModelingLogs,
|
||||||
});
|
});
|
||||||
|
|
||||||
// NOTE: This pet type should have been looked up when syncing pet type, so
|
// NOTE: This pet type should have been looked up when syncing pet type, so
|
||||||
|
@ -107,21 +125,39 @@ async function savePetTypeAndStateModelingData(
|
||||||
if (swfAssets.length === 0) {
|
if (swfAssets.length === 0) {
|
||||||
throw new Error(`pet state ${petState.id} has no saved assets?`);
|
throw new Error(`pet state ${petState.id} has no saved assets?`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const relationshipInserts = swfAssets.map((sa) => ({
|
||||||
|
parentType: "PetState",
|
||||||
|
parentId: petState.id,
|
||||||
|
swfAssetId: sa.id,
|
||||||
|
}));
|
||||||
|
|
||||||
const qs = swfAssets.map((_) => `(?, ?, ?)`).join(", ");
|
const qs = swfAssets.map((_) => `(?, ?, ?)`).join(", ");
|
||||||
const values = swfAssets
|
const values = relationshipInserts
|
||||||
.map((sa) => ["PetState", petState.id, sa.id])
|
.map(({ parentType, parentId, swfAssetId }) => [
|
||||||
|
parentType,
|
||||||
|
parentId,
|
||||||
|
swfAssetId,
|
||||||
|
])
|
||||||
.flat();
|
.flat();
|
||||||
await db.execute(
|
await db.execute(
|
||||||
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
|
`INSERT INTO parents_swf_assets (parent_type, parent_id, swf_asset_id)
|
||||||
VALUES ${qs};`,
|
VALUES ${qs};`,
|
||||||
values
|
values
|
||||||
);
|
);
|
||||||
|
|
||||||
|
addToModelingLogs({
|
||||||
|
tableName: "parents_swf_assets",
|
||||||
|
inserts: relationshipInserts,
|
||||||
|
updates: [],
|
||||||
|
});
|
||||||
},
|
},
|
||||||
|
addToModelingLogs,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function saveItemModelingData(customPetData, context) {
|
async function saveItemModelingData(customPetData, context) {
|
||||||
const { db, itemLoader, itemTranslationLoader } = context;
|
const { db, itemLoader, itemTranslationLoader, addToModelingLogs } = context;
|
||||||
|
|
||||||
const objectInfos = Object.values(customPetData.object_info_registry);
|
const objectInfos = Object.values(customPetData.object_info_registry);
|
||||||
const incomingItems = objectInfos.map((objectInfo) => ({
|
const incomingItems = objectInfos.map((objectInfo) => ({
|
||||||
|
@ -148,6 +184,7 @@ async function saveItemModelingData(customPetData, context) {
|
||||||
tableName: "items",
|
tableName: "items",
|
||||||
buildLoaderKey: (row) => row.id,
|
buildLoaderKey: (row) => row.id,
|
||||||
buildUpdateCondition: (row) => [`id = ?`, row.id],
|
buildUpdateCondition: (row) => [`id = ?`, row.id],
|
||||||
|
addToModelingLogs,
|
||||||
}),
|
}),
|
||||||
syncToDb(db, incomingItemTranslations, {
|
syncToDb(db, incomingItemTranslations, {
|
||||||
loader: itemTranslationLoader,
|
loader: itemTranslationLoader,
|
||||||
|
@ -157,12 +194,13 @@ async function saveItemModelingData(customPetData, context) {
|
||||||
`item_id = ? AND locale = "en"`,
|
`item_id = ? AND locale = "en"`,
|
||||||
row.itemId,
|
row.itemId,
|
||||||
],
|
],
|
||||||
|
addToModelingLogs,
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function saveSwfAssetModelingData(customPetData, context) {
|
async function saveSwfAssetModelingData(customPetData, context) {
|
||||||
const { db, swfAssetByRemoteIdLoader } = context;
|
const { db, swfAssetByRemoteIdLoader, addToModelingLogs } = context;
|
||||||
|
|
||||||
const objectAssets = Object.values(customPetData.object_asset_registry);
|
const objectAssets = Object.values(customPetData.object_asset_registry);
|
||||||
const incomingItemSwfAssets = objectAssets.map((objectAsset) => ({
|
const incomingItemSwfAssets = objectAssets.map((objectAsset) => ({
|
||||||
|
@ -244,6 +282,12 @@ async function saveSwfAssetModelingData(customPetData, context) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const relationshipInserts = itemAssetInserts.map(({ remoteId }) => ({
|
||||||
|
parentType: "Item",
|
||||||
|
parentId: assetIdToItemIdMap.get(remoteId),
|
||||||
|
remoteId,
|
||||||
|
}));
|
||||||
|
|
||||||
const qs = itemAssetInserts
|
const qs = itemAssetInserts
|
||||||
.map(
|
.map(
|
||||||
(_) =>
|
(_) =>
|
||||||
|
@ -254,11 +298,11 @@ async function saveSwfAssetModelingData(customPetData, context) {
|
||||||
`(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))`
|
`(SELECT id FROM swf_assets WHERE type = "object" AND remote_id = ?))`
|
||||||
)
|
)
|
||||||
.join(", ");
|
.join(", ");
|
||||||
const values = itemAssetInserts
|
const values = relationshipInserts
|
||||||
.map(({ remoteId: swfAssetId }) => [
|
.map(({ parentType, parentId, remoteId }) => [
|
||||||
"Item",
|
parentType,
|
||||||
assetIdToItemIdMap.get(swfAssetId),
|
parentId,
|
||||||
swfAssetId,
|
remoteId,
|
||||||
])
|
])
|
||||||
.flat();
|
.flat();
|
||||||
|
|
||||||
|
@ -267,7 +311,14 @@ async function saveSwfAssetModelingData(customPetData, context) {
|
||||||
VALUES ${qs}`,
|
VALUES ${qs}`,
|
||||||
values
|
values
|
||||||
);
|
);
|
||||||
|
|
||||||
|
addToModelingLogs({
|
||||||
|
tableName: "parents_swf_assets",
|
||||||
|
inserts: relationshipInserts,
|
||||||
|
updates: [],
|
||||||
|
});
|
||||||
},
|
},
|
||||||
|
addToModelingLogs,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -294,6 +345,7 @@ async function syncToDb(
|
||||||
includeCreatedAt = true,
|
includeCreatedAt = true,
|
||||||
includeUpdatedAt = true,
|
includeUpdatedAt = true,
|
||||||
afterInsert = null,
|
afterInsert = null,
|
||||||
|
addToModelingLogs,
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
const loaderKeys = incomingRows.map(buildLoaderKey);
|
const loaderKeys = incomingRows.map(buildLoaderKey);
|
||||||
|
@ -411,6 +463,14 @@ async function syncToDb(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
await Promise.all(updatePromises);
|
await Promise.all(updatePromises);
|
||||||
|
|
||||||
|
if (inserts.length > 0 || updates.length > 0) {
|
||||||
|
addToModelingLogs({
|
||||||
|
tableName,
|
||||||
|
inserts,
|
||||||
|
updates,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { saveModelingData };
|
module.exports = { saveModelingData };
|
||||||
|
|
Loading…
Reference in a new issue