cache zone data

I noticed that, while looking up zone data from the db is near instant when you're on the same box, it's like 300ms here!

In this change, we start downloading zone data into the build process. That way, we can have a very fast and practically-up-to-date cache (I'm not sure I've changed it in many years), while being confident that it's in sync with the database source of truth (for things like join queries).

build-cached-data.js

@@ -0,0 +1,49 @@
+const fs = require("fs").promises;
+const path = require("path");
+
+const connectToDb = require("./src/server/db");
+const { normalizeRow } = require("./src/server/util");
+
+const cachedDataPath = path.join(__dirname, "build", "cached-data");
+
+async function buildZonesCache(db) {
+  const [rows] = await db.query(`SELECT * FROM zones;`);
+  const entities = rows.map(normalizeRow);
+
+  const filePath = path.join(cachedDataPath, "zones.json");
+  fs.writeFile(filePath, JSON.stringify(entities, null, 4), "utf8");
+
+  console.log(`📚 Wrote zones to ${path.relative(process.cwd(), filePath)}`);
+}
+
+async function buildZoneTranslationsCache(db) {
+  const [rows] = await db.query(
+    `SELECT * FROM zone_translations WHERE locale = "en";`
+  );
+  const entities = rows.map(normalizeRow);
+
+  const filePath = path.join(cachedDataPath, "zone_translations.json");
+  fs.writeFile(filePath, JSON.stringify(entities, null, 4), "utf8");
+
+  console.log(
+    `📚 Wrote zone translations to ${path.relative(process.cwd(), filePath)}`
+  );
+}
+
+async function main() {
+  const db = await connectToDb();
+  await fs.mkdir(cachedDataPath, { recursive: true });
+
+  try {
+    await Promise.all([buildZonesCache(db), buildZoneTranslationsCache(db)]);
+  } catch (e) {
+    db.close();
+    throw e;
+  }
+  db.close();
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exitCode = 1;
+});

package.json

@@ -32,8 +32,9 @@
     "react-transition-group": "^4.3.0"
   },
   "scripts": {
-    "start": "react-app-rewired start",
+    "start": "yarn build-cached-data && react-app-rewired start",
-    "build": "react-app-rewired build",
+    "build-cached-data": "node -r dotenv/config build-cached-data.js",
+    "build": "react-app-rewired build && yarn build-cached-data",
     "test": "react-app-rewired test --env=jsdom",
     "eject": "react-scripts eject",
     "setup-mysql-user": "mysql -h impress.openneo.net -u matchu -p < setup-mysql-user.sql",

src/server/index.js

@@ -1,5 +1,5 @@
 const { gql, makeExecutableSchema } = require("apollo-server");
-import { addBeelineToSchema, beelinePlugin } from "./lib/beeline-graphql";
+const { addBeelineToSchema, beelinePlugin } = require("./lib/beeline-graphql");
 
 const connectToDb = require("./db");
 const buildLoaders = require("./loaders");
@@ -12,6 +12,15 @@ const {
   getGenderPresentation,
 } = require("./util");
 
+// These are caches of stable database tables. They're built in the
+// `build-cached-data` script, at build time and dev-start time.
+const zoneRows = require("../../build/cached-data/zones.json");
+const zones = new Map(zoneRows.map((z) => [z.id, z]));
+const zoneTranslationRows = require("../../build/cached-data/zone_translations.json");
+const zoneTranslations = new Map(
+  zoneTranslationRows.map((zt) => [`${zt.zoneId}-${zt.locale}`, zt])
+);
+
 const typeDefs = gql`
   directive @cacheControl(maxAge: Int!) on FIELD_DEFINITION | OBJECT
 
@@ -344,10 +353,9 @@ const resolvers = {
       const layer = await swfAssetLoader.load(id);
       return layer.bodyId;
     },
-    zone: async ({ id }, _, { swfAssetLoader, zoneLoader }) => {
+    zone: async ({ id }, _, { swfAssetLoader }) => {
       const layer = await swfAssetLoader.load(id);
-      const zone = await zoneLoader.load(layer.zoneId);
+      return { id: layer.zoneId };
-      return zone;
     },
     swfUrl: async ({ id }, _, { swfAssetLoader }) => {
       const layer = await swfAssetLoader.load(id);
@@ -424,17 +432,8 @@ const resolvers = {
     },
   },
   Zone: {
-    depth: async ({ id }, _, { zoneLoader }) => {
+    depth: ({ id }) => zones.get(id).depth,
-      // TODO: Should we extend this loader-in-field pattern elsewhere? I like
+    label: ({ id }) => zoneTranslations.get(`${id}-en`).label,
-      //       that we avoid the fetch in cases where we only want the zone ID,
-      //       but it adds complexity 🤔
-      const zone = await zoneLoader.load(id);
-      return zone.depth;
-    },
-    label: async ({ id }, _, { zoneTranslationLoader }) => {
-      const zoneTranslation = await zoneTranslationLoader.load(id);
-      return zoneTranslation.label;
-    },
   },
   Color: {
     name: async ({ id }, _, { colorTranslationLoader }) => {

src/server/loaders.js

@@ -370,42 +370,6 @@ const buildPetStatesForPetTypeLoader = (db, loaders) =>
     );
   });
 
-const buildZoneLoader = (db) =>
-  new DataLoader(async (ids) => {
-    const qs = ids.map((_) => "?").join(",");
-    const [rows, _] = await db.execute(
-      `SELECT * FROM zones WHERE id IN (${qs})`,
-      ids
-    );
-
-    const entities = rows.map(normalizeRow);
-    const entitiesById = new Map(entities.map((e) => [e.id, e]));
-
-    return ids.map(
-      (id) =>
-        entitiesById.get(String(id)) ||
-        new Error(`could not find zone with ID: ${id}`)
-    );
-  });
-
-const buildZoneTranslationLoader = (db) =>
-  new DataLoader(async (zoneIds) => {
-    const qs = zoneIds.map((_) => "?").join(",");
-    const [rows, _] = await db.execute(
-      `SELECT * FROM zone_translations WHERE zone_id IN (${qs}) AND locale = "en"`,
-      zoneIds
-    );
-
-    const entities = rows.map(normalizeRow);
-    const entitiesByZoneId = new Map(entities.map((e) => [e.zoneId, e]));
-
-    return zoneIds.map(
-      (zoneId) =>
-        entitiesByZoneId.get(String(zoneId)) ||
-        new Error(`could not find translation for zone ${zoneId}`)
-    );
-  });
-
 function buildLoaders(db) {
   const loaders = {};
   loaders.loadAllSpecies = loadAllSpecies(db);
@@ -435,8 +399,6 @@ function buildLoaders(db) {
     loaders
   );
   loaders.speciesTranslationLoader = buildSpeciesTranslationLoader(db);
-  loaders.zoneLoader = buildZoneLoader(db);
-  loaders.zoneTranslationLoader = buildZoneTranslationLoader(db);
 
   return loaders;
 }

src/server/query-tests/Item.test.js

@@ -132,22 +132,6 @@ describe("Item", () => {
             "180",
           ],
         ],
-        Array [
-          "SELECT * FROM zones WHERE id IN (?,?,?)",
-          Array [
-            "26",
-            "40",
-            "3",
-          ],
-        ],
-        Array [
-          "SELECT * FROM zone_translations WHERE zone_id IN (?,?,?) AND locale = \\"en\\"",
-          Array [
-            "26",
-            "40",
-            "3",
-          ],
-        ],
       ]
     `);
   });

src/server/query-tests/PetAppearance.test.js

@@ -82,17 +82,6 @@ describe("PetAppearance", () => {
             "75",
           ],
         ],
-        Array [
-          "SELECT * FROM zones WHERE id IN (?,?,?,?,?,?)",
-          Array [
-            "15",
-            "5",
-            "37",
-            "30",
-            "33",
-            "34",
-          ],
-        ],
       ]
     `);
   });
@@ -179,17 +168,6 @@ describe("PetAppearance", () => {
             "75",
           ],
         ],
-        Array [
-          "SELECT * FROM zones WHERE id IN (?,?,?,?,?,?)",
-          Array [
-            "15",
-            "5",
-            "37",
-            "30",
-            "33",
-            "34",
-          ],
-        ],
       ]
     `);
   });