Save hopefully ~$200/month (#183)

* Save hopefully ~$200/month

Turns out that this consumes a lot of bandwidth from RTDB, I'm seeing like 8 GB/day. Which is kind of insane :(

I think the bandwidth is being counted even for rows that are never returned from the query due to being filtered out. Firebase is just really inefficient. So it's iterating through basically the entire `gameData` database field every time I run this function, ugh.

Making some changes to reduce expenditure.

* Actually save money by improving indexing

Why doesn't Firebase have query iterators…

Author: ekzhang

database.rules.json

@@ -28,6 +28,7 @@
       }
     },
     "gameData": {
+      ".indexOn": "populatedAt",
       "$gameId": {
         ".read": "auth != null",
         "events": {

functions/src/index.ts

@@ -7,7 +7,7 @@ import PQueue from "p-queue";
 import Stripe from "stripe";
 
 import { GameMode, findSet, generateDeck, replayEvents } from "./game";
-import { databaseIterator, gzip } from "./utils";
+import { gzip } from "./utils";
 
 initializeApp(); // Sets the default Firebase app.
 
@@ -511,10 +511,24 @@ export const archiveStaleGames = functions
     const cutoff = Date.now() - 14 * 86400 * 1000; // 14 days ago
     const queue = new PQueue({ concurrency: 200 });
 
-    for await (const [gameId, gameState] of databaseIterator("gameData")) {
+    const snap = await getDatabase()
+      .ref("gameData")
+      .orderByChild("populatedAt")
+      .endBefore(cutoff)
+      .get();
+
+    const childKeys: string[] = [];
+    snap.forEach((child) => {
+      childKeys.push(child.key);
+    });
+
+    let archiveCount = 0;
+    for (const gameId of childKeys) {
+      const gameState = snap.child(gameId);
       const populatedAt: number | null = gameState.child("populatedAt").val();
       if (!populatedAt || populatedAt < cutoff) {
         await queue.onEmpty();
+        archiveCount += 1;
         queue.add(async () => {
           console.log(`Archiving stale game state for ${gameId}`);
           await archiveGameState(gameId, gameState);
@@ -523,4 +537,5 @@ export const archiveStaleGames = functions
     }
 
     await queue.onIdle();
+    console.log(`Completed archive of ${archiveCount} games`);
   });

functions/src/utils.ts

@@ -30,21 +30,22 @@ export const gzip = {
 export async function* databaseIterator(
   path: string,
   batchSize = 1000,
+  start?: string, // inclusive
+  end?: string, // inclusive
 ): AsyncGenerator<[string, DataSnapshot]> {
-  let lastKey = null;
+  let lastKey: string | undefined = undefined;
   while (true) {
-    const snap = lastKey
-      ? await getDatabase()
-          .ref(path)
-          .orderByKey()
-          .startAfter(lastKey)
-          .limitToFirst(batchSize)
-          .get()
-      : await getDatabase()
-          .ref(path)
-          .orderByKey()
-          .limitToFirst(batchSize)
-          .get();
+    let query = getDatabase().ref(path).orderByKey();
+    if (lastKey !== undefined) {
+      query = query.startAfter(lastKey);
+    } else if (start !== undefined) {
+      query = query.startAt(start);
+    }
+    if (end !== undefined) {
+      query = query.endAt(end);
+    }
+
+    const snap = await query.limitToFirst(batchSize).get();
     if (!snap.exists()) return;
 
     const childKeys: string[] = [];

scripts/src/utils.js

@@ -6,23 +6,29 @@ import { getDatabase } from "firebase-admin/database";
  *
  * @param {string} path The path to the reference to iterate through.
  * @param {number} batchSize The number of children to fetch in each batch.
+ * @param {string} [start] The key to start at (inclusive).
+ * @param {string} [end] The key to end at (inclusive).
  * @returns {AsyncGenerator<[string, import("firebase-admin/database").DataSnapshot]>}
  */
-export async function* databaseIterator(path, batchSize = 1000) {
-  let lastKey = null;
+export async function* databaseIterator(
+  path,
+  batchSize = 1000,
+  start, // inclusive
+  end, // inclusive
+) {
+  let lastKey = undefined;
   while (true) {
-    const snap = lastKey
-      ? await getDatabase()
-          .ref(path)
-          .orderByKey()
-          .startAfter(lastKey)
-          .limitToFirst(batchSize)
-          .get()
-      : await getDatabase()
-          .ref(path)
-          .orderByKey()
-          .limitToFirst(batchSize)
-          .get();
+    let query = getDatabase().ref(path).orderByKey();
+    if (lastKey !== undefined) {
+      query = query.startAfter(lastKey);
+    } else if (start !== undefined) {
+      query = query.startAt(start);
+    }
+    if (end !== undefined) {
+      query = query.endAt(end);
+    }
+
+    const snap = await query.limitToFirst(batchSize).get();
     if (!snap.exists()) return;
 
     const childKeys = [];