update modeled item data

src/server/query-tests/Pet.test.js

@@ -1,5 +1,11 @@
 const gql = require("graphql-tag");
-const { query, getDbCalls, clearDbCalls, useTestDb } = require("./setup.js");
+const {
+  query,
+  getDbCalls,
+  clearDbCalls,
+  useTestDb,
+  connectToDb,
+} = require("./setup.js");
 
 describe("Pet", () => {
   it("looks up a pet", async () => {
@@ -106,35 +112,90 @@ describe("Pet", () => {
 
     expect(res2).toHaveNoErrors();
     expect(res2.data).toMatchSnapshot();
-    expect(getDbCalls()).toMatchInlineSnapshot(`
+    expect(getDbCalls()).toMatchSnapshot();
-      Array [
+  });
-        Array [
+
-          "SELECT * FROM item_translations WHERE item_id IN (?,?,?,?,?,?,?,?) AND locale = \\"en\\"",
+  it("models updated item data", async () => {
-          Array [
+    useTestDb();
-            "37229",
+
-            "37375",
+    // First, write a fake version of the Jewelled Staff to the database.
-            "38911",
+    // It's mostly the real data, except we changed rarity_index,
-            "38912",
+    // thumbnail_url, translated name, and translated description.
-            "38913",
+    const db = await connectToDb();
-            "43014",
+    await Promise.all([
-            "43397",
+      db.query(
-            "48313",
+        `INSERT INTO items (id, zones_restrict, thumbnail_url, category,
-          ],
+                            type, rarity_index, price, weight_lbs)
-        ],
+           VALUES (43397, "00000000000000000000000000000000000000000000000",
-        Array [
+                   "http://example.com/favicon.ico", "Clothes", "Clothes", 101,
-          "SELECT * FROM items WHERE id IN (?,?,?,?,?,?,?,?)",
+                   0, 1);`
-          Array [
+      ),
-            "37229",
+      db.query(
-            "37375",
+        `INSERT INTO item_translations (item_id, locale, name, description,
-            "38911",
+                                        rarity)
-            "38912",
+           VALUES (43397, "en", "Bejewelled Staffo",
-            "38913",
+                   "This staff is really neat and good!", "Artifact")`
-            "43014",
+      ),
-            "43397",
+    ]);
-            "48313",
+
-          ],
+    clearDbCalls();
-        ],
+
-      ]
+    // Then, load a pet wearing this. It should trigger an UPDATE for the item
-    `);
+    // and its translation, and return the new names in the query.
+    const res = await query({
+      query: gql`
+        query {
+          petOnNeopetsDotCom(petName: "roopal27") {
+            items {
+              id
+              name
+              description
+              thumbnailUrl
+              rarityIndex
+            }
+          }
+        }
+      `,
+    });
+
+    expect(res).toHaveNoErrors();
+    const itemData = res.data.petOnNeopetsDotCom.items.find(
+      (item) => item.id === "43397"
+    );
+    expect(itemData).toEqual({
+      id: "43397",
+      name: "Jewelled Staff",
+      description: "This jewelled staff shines with a magical light.",
+      thumbnailUrl: "http://images.neopets.com/items/mall_staff_jewelled.gif",
+      rarityIndex: 500,
+    });
+    expect(getDbCalls()).toMatchSnapshot();
+
+    clearDbCalls();
+
+    // Finally, load the item. It should have the updated values.
+    const res2 = await query({
+      query: gql`
+        query {
+          item(id: "43397") {
+            id
+            name
+            description
+            thumbnailUrl
+            rarityIndex
+          }
+        }
+      `,
+    });
+
+    expect(res2).toHaveNoErrors();
+    expect(res2.data.item).toEqual({
+      id: "43397",
+      name: "Jewelled Staff",
+      description: "This jewelled staff shines with a magical light.",
+      thumbnailUrl: "http://images.neopets.com/items/mall_staff_jewelled.gif",
+      rarityIndex: 500,
+    });
+    expect(getDbCalls()).toMatchSnapshot();
   });
 });

src/server/query-tests/__snapshots__/Pet.test.js.snap

@@ -410,3 +410,230 @@ Object {
   ],
 }
 `;
+
+exports[`Pet models new item data 4`] = `
+Array [
+  Array [
+    "SELECT * FROM item_translations WHERE item_id IN (?,?,?,?,?,?,?,?) AND locale = \\"en\\"",
+    Array [
+      "37229",
+      "37375",
+      "38911",
+      "38912",
+      "38913",
+      "43014",
+      "43397",
+      "48313",
+    ],
+  ],
+  Array [
+    "SELECT * FROM items WHERE id IN (?,?,?,?,?,?,?,?)",
+    Array [
+      "37229",
+      "37375",
+      "38911",
+      "38912",
+      "38913",
+      "43014",
+      "43397",
+      "48313",
+    ],
+  ],
+]
+`;
+
+exports[`Pet models updated item data 1`] = `
+Array [
+  Array [
+    "SELECT * FROM items WHERE id IN (?,?,?,?,?,?,?,?)",
+    Array [
+      "37229",
+      "37375",
+      "38911",
+      "38912",
+      "38913",
+      "43014",
+      "43397",
+      "48313",
+    ],
+  ],
+  Array [
+    "SELECT * FROM item_translations WHERE item_id IN (?,?,?,?,?,?,?,?) AND locale = \\"en\\"",
+    Array [
+      "37229",
+      "37375",
+      "38911",
+      "38912",
+      "38913",
+      "43014",
+      "43397",
+      "48313",
+    ],
+  ],
+  Array [
+    "INSERT INTO items (category, created_at, id, price, rarity_index, thumbnail_url, type, updated_at, weight_lbs, zones_restrict) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);",
+    Array [
+      "Gift",
+      2020-01-01T00:00:00.000Z,
+      "37229",
+      0,
+      101,
+      "http://images.neopets.com/items/gif_magicball_table.gif",
+      "Gift",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000000000000000000000000000000",
+      "Special",
+      2020-01-01T00:00:00.000Z,
+      "37375",
+      209,
+      75,
+      "http://images.neopets.com/items/bg_moonstars.gif",
+      "Mystical Surroundings",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000000000000000000000000000000000000",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      "38911",
+      980,
+      92,
+      "http://images.neopets.com/items/clo_zafara_agent_hood.gif",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000000000000000000001100000000000000",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      "38912",
+      1476,
+      90,
+      "http://images.neopets.com/items/clo_zafara_agent_robe.gif",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000101000000000000000000000000000000",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      "38913",
+      1177,
+      88,
+      "http://images.neopets.com/items/clo_zafara_agent_gloves.gif",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000000000000000000000000000000000000",
+      "Toys",
+      2020-01-01T00:00:00.000Z,
+      "43014",
+      1033,
+      80,
+      "http://images.neopets.com/items/toy_stringlight_illleaf.gif",
+      "Toy",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000000000000000000000000000000",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      "48313",
+      0,
+      101,
+      "http://images.neopets.com/items/clo_altcuplogo_brooch.gif",
+      "Clothes",
+      2020-01-01T00:00:00.000Z,
+      1,
+      "0000000000000000000000000000000000000000000000000000",
+    ],
+  ],
+  Array [
+    "INSERT INTO item_translations (created_at, description, item_id, locale, name, rarity, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?);",
+    Array [
+      2020-01-01T00:00:00.000Z,
+      "What does this ball actually do?",
+      "37229",
+      "en",
+      "Magic Ball Table",
+      "Special",
+      2020-01-01T00:00:00.000Z,
+      2020-01-01T00:00:00.000Z,
+      "Dont forget to wish upon a star.",
+      "37375",
+      "en",
+      "Moon and Stars Background",
+      "Uncommon",
+      2020-01-01T00:00:00.000Z,
+      2020-01-01T00:00:00.000Z,
+      "Hide your face and hair so no one can recognise you.",
+      "38911",
+      "en",
+      "Zafara Agent Hood",
+      "Very Rare",
+      2020-01-01T00:00:00.000Z,
+      2020-01-01T00:00:00.000Z,
+      "This robe is great for being stealthy.",
+      "38912",
+      "en",
+      "Zafara Agent Robe",
+      "Very Rare",
+      2020-01-01T00:00:00.000Z,
+      2020-01-01T00:00:00.000Z,
+      "Dont leave any trace that you were there with these gloves.",
+      "38913",
+      "en",
+      "Zafara Agent Gloves",
+      "Rare",
+      2020-01-01T00:00:00.000Z,
+      2020-01-01T00:00:00.000Z,
+      "These leaves almost look magical with their gentle glow.",
+      "43014",
+      "en",
+      "Green Leaf String Lights",
+      "Uncommon",
+      2020-01-01T00:00:00.000Z,
+      2020-01-01T00:00:00.000Z,
+      "Even the announcers of the Altador Cup celebrate.  This was given out by the Advent Calendar in Y11.",
+      "48313",
+      "en",
+      "Altador Cup Brooch",
+      "Special",
+      2020-01-01T00:00:00.000Z,
+    ],
+  ],
+  Array [
+    "UPDATE items SET rarity_index = ?, thumbnail_url = ?, updated_at = ?, zones_restrict = ? WHERE id = ?;",
+    Array [
+      500,
+      "http://images.neopets.com/items/mall_staff_jewelled.gif",
+      2020-01-01T00:00:00.000Z,
+      "0000000000000000000000000000000000000000000000",
+      "43397",
+    ],
+  ],
+  Array [
+    "UPDATE item_translations SET description = ?, name = ?, updated_at = ? WHERE item_id = ? AND locale = \\"en\\";",
+    Array [
+      "This jewelled staff shines with a magical light.",
+      "Jewelled Staff",
+      2020-01-01T00:00:00.000Z,
+      "43397",
+    ],
+  ],
+]
+`;
+
+exports[`Pet models updated item data 2`] = `
+Array [
+  Array [
+    "SELECT * FROM item_translations WHERE item_id IN (?) AND locale = \\"en\\"",
+    Array [
+      "43397",
+    ],
+  ],
+  Array [
+    "SELECT * FROM items WHERE id IN (?)",
+    Array [
+      "43397",
+    ],
+  ],
+]
+`;

src/server/query-tests/setup.js

@@ -83,6 +83,7 @@ beforeEach(() => {
   }
   dbEnvironment = "production";
   dbSetupDone = false;
+  db = null;
 });
 afterAll(() => {
   if (db) {
@@ -138,7 +139,7 @@ module.exports = {
   query,
   getDbCalls,
   clearDbCalls,
-  getDb: () => db,
+  connectToDb,
   useTestDb,
   logInAsTestUser,
 };

src/server/types/Outfit.js

@@ -157,9 +157,7 @@ async function saveModelingData(
 ) {
   const objectInfos = Object.values(customPetData.object_info_registry);
 
-  const incomingItems = objectInfos.map((objectInfo) => [
+  const incomingItems = objectInfos.map((objectInfo) => ({
-    String(objectInfo.obj_info_id),
-    {
     id: String(objectInfo.obj_info_id),
     zonesRestrict: objectInfo.zones_restrict,
     thumbnailUrl: objectInfo.thumbnail_url,
@@ -168,28 +166,32 @@ async function saveModelingData(
     rarityIndex: objectInfo.rarity_index,
     price: objectInfo.price,
     weightLbs: objectInfo.weight_lbs,
-    },
+  }));
-  ]);
 
-  const incomingItemTranslations = objectInfos.map((objectInfo) => [
+  const incomingItemTranslations = objectInfos.map((objectInfo) => ({
-    String(objectInfo.obj_info_id),
-    {
     itemId: String(objectInfo.obj_info_id),
     locale: "en",
     name: objectInfo.name,
     description: objectInfo.description,
     rarity: objectInfo.rarity,
-    },
+  }));
-  ]);
 
   await Promise.all([
-    syncToDb("items", itemLoader, db, incomingItems),
+    syncToDb(db, incomingItems, {
-    syncToDb(
+      loader: itemLoader,
-      "item_translations",
+      tableName: "items",
-      itemTranslationLoader,
+      buildLoaderKey: (row) => row.id,
-      db,
+      buildUpdateCondition: (row) => [`id = ?`, row.id],
-      incomingItemTranslations
+    }),
-    ),
+    syncToDb(db, incomingItemTranslations, {
+      loader: itemTranslationLoader,
+      tableName: "item_translations",
+      buildLoaderKey: (row) => row.itemId,
+      buildUpdateCondition: (row) => [
+        `item_id = ? AND locale = "en"`,
+        row.itemId,
+      ],
+    }),
   ]);
 }
 
@@ -205,42 +207,84 @@ async function saveModelingData(
  * Will perform one call to the loader, and at most one INSERT, and at most one
  * UPDATE, regardless of how many rows we're syncing.
  */
-async function syncToDb(tableName, loader, db, incomingRows) {
+async function syncToDb(
-  const loaderKeys = incomingRows.map(([key, _]) => key);
+  db,
+  incomingRows,
+  { loader, tableName, buildLoaderKey, buildUpdateCondition }
+) {
+  const loaderKeys = incomingRows.map(buildLoaderKey);
   const currentRows = await loader.loadMany(loaderKeys);
 
-  const rowsToInsert = [];
+  const inserts = [];
+  const updates = [];
   for (const index in incomingRows) {
-    const [_, incomingRow] = incomingRows[index];
+    const incomingRow = incomingRows[index];
     const currentRow = currentRows[index];
 
+    // If there is no corresponding row in the database, prepare an insert.
     if (currentRow instanceof Error) {
-      rowsToInsert.push({
+      inserts.push({
         ...incomingRow,
         createdAt: new Date(),
         updatedAt: new Date(),
       });
+      continue;
+    }
+
+    // If there's a row in the database, and some of the values don't match,
+    // prepare an update with the updated fields only.
+    const updatedKeys = Object.keys(incomingRow).filter(
+      (k) => incomingRow[k] !== currentRow[k]
+    );
+    if (updatedKeys.length > 0) {
+      const update = {};
+      for (const key of updatedKeys) {
+        update[key] = incomingRow[key];
+      }
+      update.updatedAt = new Date();
+      updates.push({ incomingRow, update });
     }
   }
 
-  if (rowsToInsert.length > 0) {
+  // Do a bulk insert of anything that needs added.
+  if (inserts.length > 0) {
     // Get the column names from the first row, and convert them to
     // underscore-case instead of camel-case.
-    const rowKeys = Object.keys(rowsToInsert[0]).sort();
+    const rowKeys = Object.keys(inserts[0]).sort();
     const columnNames = rowKeys.map((key) =>
       key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase())
     );
     const columnsStr = columnNames.join(", ");
     const qs = columnNames.map((_) => "?").join(", ");
-    const rowQs = rowsToInsert.map((_) => "(" + qs + ")").join(", ");
+    const rowQs = inserts.map((_) => "(" + qs + ")").join(", ");
-    const rowFields = rowsToInsert.map((row) => rowKeys.map((key) => row[key]));
+    const rowFields = inserts.map((row) => rowKeys.map((key) => row[key]));
     await db.execute(
       `INSERT INTO ${tableName} (${columnsStr}) VALUES ${rowQs};`,
       rowFields.flat()
     );
   }
 
-  // TODO: Update rows that need updating
+  // Do parallel updates of anything that needs updated.
+  // NOTE: I feel like it's not possible to do bulk updates, even in a
+  //       multi-statement mysql2 request? I might be wrong, but whatever; it's
+  //       very uncommon, and any perf hit would be nbd.
+  const updatePromises = [];
+  for (const { incomingRow, update } of updates) {
+    const rowKeys = Object.keys(update).sort();
+    const rowValues = rowKeys.map((k) => update[k]);
+    const columnNames = rowKeys.map((key) =>
+      key.replace(/[A-Z]/g, (m) => "_" + m[0].toLowerCase())
+    );
+    const qs = columnNames.map((c) => `${c} = ?`).join(", ");
+    const [conditionQs, ...conditionValues] = buildUpdateCondition(incomingRow);
+    updatePromises.push(
+      db.execute(`UPDATE ${tableName} SET ${qs} WHERE ${conditionQs};`, [
+        ...rowValues,
+        ...conditionValues,
+      ])
+    );
+  }
+  await Promise.all(updatePromises);
 }
 
 module.exports = { typeDefs, resolvers };