feat: add vercel blobs to cache historical subgraph data

libs/src/oracle-sdk-v2/services/oraclev1/gql/queries.ts

@@ -16,8 +16,29 @@ export async function getPriceRequests(
 ) {
   const chainName = chainsById[chainId];
   const queryName = makeQueryName(oracleType, chainName);
-  const result = await fetchAllRequests(url, queryName, oracleType);
-  return result;
+
+  const result = (await fetch(
+    `/api/subgraph-blob-info?url=${url}&queryName=${queryName}&oracleType=${oracleType}`,
+  ).then((res) => res.json())) as { url: string | null };
+
+  if (result.url === null) {
+    const requests = await fetchAllRequests(url, queryName, oracleType);
+    return requests;
+  } else {
+    const blob = await fetch(result.url);
+    let requests = (await blob.json()) as OOV1GraphEntity[] | OOV2GraphEntity[];
+    if (requests.length > 0) {
+      // Blob should cover most historical data, but we need to fetch the rest.
+      const remainingRequests = await fetchAllRequests(
+        url,
+        queryName,
+        oracleType,
+        Number(requests[0].time),
+      );
+      requests = [...remainingRequests, ...requests];
+    }
+    return requests;
+  }
 }
 
 export async function* getPriceRequestsIncremental(
@@ -75,17 +96,18 @@ async function* fetchAllRequestsIncremental(
     }
   }
 }
-async function fetchAllRequests(
+export async function fetchAllRequests(
   url: string,
   queryName: string,
   oracleType: OracleType,
+  after?: number,
 ) {
   const result: (OOV1GraphEntity | OOV2GraphEntity)[] = [];
   let skip = 0;
   const first = 1000;
   let requests = await fetchPriceRequests(
     url,
-    makeQuery(queryName, oracleType, first, skip),
+    makeQuery(queryName, oracleType, first, skip, after),
   );
 
   // thegraph wont allow skip > 5000,
@@ -94,7 +116,7 @@ async function fetchAllRequests(
     skip += first;
     requests = await fetchPriceRequests(
       url,
-      makeQuery(queryName, oracleType, first, skip),
+      makeQuery(queryName, oracleType, first, skip, after),
     );
   }
 
@@ -104,7 +126,7 @@ async function fetchAllRequests(
     const lastTime = requests[requests.length - 1].time;
     requests = await fetchPriceRequests(
       url,
-      makeTimeBasedQuery(queryName, oracleType, first, Number(lastTime)),
+      makeTimeBasedQuery(queryName, oracleType, first, Number(lastTime), after),
     );
   }
 
@@ -113,7 +135,7 @@ async function fetchAllRequests(
   return result;
 }
 
-async function fetchPriceRequests(url: string, query: string) {
+export async function fetchPriceRequests(url: string, query: string) {
   const result = await request<
     PriceRequestsQuery | { errors: { message: string }[] }
   >(url, query);
@@ -123,15 +145,18 @@ async function fetchPriceRequests(url: string, query: string) {
   return result.optimisticPriceRequests;
 }
 
-function makeQuery(
+export function makeQuery(
   queryName: string,
   oracleType: OracleType,
   first: number,
   skip: number,
+  after?: number,
 ) {
   const query = gql`
   query ${queryName} {
-    optimisticPriceRequests(orderBy: time, orderDirection: desc, first: ${first}, skip: ${skip}) {
+    optimisticPriceRequests(orderBy: time, orderDirection: desc, first: ${first}, skip: ${skip}${
+      after ? `, where: { time_gt: ${after} }` : ""
+    }) {
       id
       identifier
       ancillaryData
@@ -189,15 +214,18 @@ function makeQuery(
   return query;
 }
 
-function makeTimeBasedQuery(
+export function makeTimeBasedQuery(
   queryName: string,
   oracleType: OracleType,
   first: number,
   lastTime: number,
+  after?: number,
 ) {
   const query = gql`
   query ${queryName} {
-    optimisticPriceRequests(orderBy: time, orderDirection: desc, first: ${first}, where: { time_lt: ${lastTime}}) {
+    optimisticPriceRequests(orderBy: time, orderDirection: desc, first: ${first}, where: { time_lt: ${lastTime}${
+      after ? `, time_gt: ${after}` : ""
+    }}) {
       id
       identifier
       ancillaryData

package.json

@@ -49,6 +49,9 @@
     "@uma/contracts-frontend": "^0.4.17",
     "@uma/contracts-node": "^0.4.17",
     "@uma/sdk": "^0.34.2",
+    "@vercel/blob": "^1.1.1",
+    "@vercel/functions": "^2.2.8",
+    "@vercel/kv": "^3.0.0",
     "autoprefixer": "^10.4.14",
     "chromatic": "^6.20.0",
     "class-variance-authority": "^0.7.1",

src/pages/api/subgraph-blob-info.ts

@@ -0,0 +1,83 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+
+import { fetchAllRequests } from "@libs/oracle-sdk-v2/services/oraclev1/gql/queries";
+import type { OracleType } from "@shared/types";
+
+import { oracleTypes } from "@shared/constants";
+import { kv } from "@vercel/kv";
+import { put } from "@vercel/blob";
+import { waitUntil } from "@vercel/functions";
+
+type BlobValue = {
+  url: string;
+  createdAt: number;
+};
+
+async function updateBlob(
+  blobName: string,
+  url: string,
+  queryName: string,
+  oracleType: OracleType,
+) {
+  const uuid = crypto.randomUUID();
+  const requests = await fetchAllRequests(url, queryName, oracleType);
+  const blobPath = `${blobName}-${uuid}.json`;
+  const { url: blobUrl } = await put(blobPath, JSON.stringify(requests), {
+    access: "public",
+    addRandomSuffix: false,
+  });
+  await kv.set(
+    blobName,
+    JSON.stringify({
+      url: blobUrl,
+      createdAt: Math.floor(Date.now() / 1000),
+    }),
+  );
+}
+
+const BLOB_TTL = 60 * 60 * 24; // 12 hours
+
+function isBlobExpired(blobValue: BlobValue) {
+  return Math.floor(Date.now() / 1000) - blobValue.createdAt > BLOB_TTL;
+}
+
+export default async function handler(
+  _request: NextApiRequest,
+  response: NextApiResponse,
+) {
+  response.setHeader("Cache-Control", "max-age=0, s-maxage=300"); // Cache for 5 minutes, reset on re-deployment.
+
+  const url = _request.query.url;
+  const queryName = _request.query.queryName;
+  const oracleType = _request.query.oracleType;
+
+  if (
+    !url ||
+    typeof url !== "string" ||
+    !queryName ||
+    typeof queryName !== "string" ||
+    !oracleType ||
+    typeof oracleType !== "string" ||
+    !oracleTypes.includes(oracleType as OracleType)
+  ) {
+    response.status(400).send({
+      message: "Missing required parameters",
+    });
+    return;
+  }
+
+  const safeUrl = url.replaceAll("/", "-"); // never allow slashes in names
+
+  const blobName = `${process.env.BLOB_KEY_PREFIX}-${safeUrl}-${queryName}-${oracleType}`;
+
+  const blobValue = await kv.get<BlobValue>(blobName);
+
+  if (blobValue === null || isBlobExpired(blobValue)) {
+    // This forces the vercel function to update the blob in the background.
+    waitUntil(updateBlob(blobName, url, queryName, oracleType as OracleType));
+  }
+
+  response.status(200).send({
+    url: blobValue?.url ?? null,
+  });
+}