feat(stats): shows ephemeral and persistent storage separately on pie…

… chart

refs #645, #665

apps/api/src/routes/v1/networkCapacity.ts

@@ -28,7 +28,13 @@ const route = createRoute({
             totalCPU: z.number(),
             totalGPU: z.number(),
             totalMemory: z.number(),
-            totalStorage: z.number()
+            totalStorage: z.number(),
+            activeEphemeralStorage: z.number(),
+            pendingEphemeralStorage: z.number(),
+            availableEphemeralStorage: z.number(),
+            activePersistentStorage: z.number(),
+            pendingPersistentStorage: z.number(),
+            availablePersistentStorage: z.number(),
           })
         }
       }

apps/api/src/routes/v1/providers/byAddress.ts

@@ -3,6 +3,11 @@ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { getProviderDetail } from "@src/services/db/providerStatusService";
 import { openApiExampleProviderAddress } from "@src/utils/constants";
 
+const statsItemSchema = () => z.object({
+  active: z.number(),
+  available: z.number(),
+  pending: z.number(),
+});
 const route = createRoute({
   method: "get",
   path: "/providers/{address}",
@@ -46,6 +51,15 @@ const route = createRoute({
             isOnline: z.boolean(),
             lastOnlineDate: z.string().nullable(),
             isAudited: z.boolean(),
+            stats: z.object({
+              cpu: statsItemSchema(),
+              gpu: statsItemSchema(),
+              memory: statsItemSchema(),
+              storage: z.object({
+                ephemeral: statsItemSchema(),
+                persistent: statsItemSchema(),
+              })
+            }),
             activeStats: z.object({
               cpu: z.number(),
               gpu: z.number(),

apps/api/src/services/db/providerStatusService.ts

@@ -1,6 +1,7 @@
 import { Provider, ProviderAttribute, ProviderAttributeSignature, ProviderSnapshotNode, ProviderSnapshotNodeGPU } from "@akashnetwork/database/dbSchemas/akash";
 import { ProviderSnapshot } from "@akashnetwork/database/dbSchemas/akash/providerSnapshot";
 import { add, sub } from "date-fns";
+import uniqBy from "lodash/uniqBy";
 import { Op } from "sequelize";
 
 import { ProviderDetail } from "@src/types/provider";
@@ -24,31 +25,58 @@ export async function getNetworkCapacity() {
     ]
   });
 
-  const filteredProviders = providers.filter((value, index, self) => self.map(x => x.hostUri).indexOf(value.hostUri) === index);
+  const filteredProviders = uniqBy(providers, provider => provider.hostUri);
+  const stats = filteredProviders.reduce(
+    (all, provider) => {
+      stats.activeCPU += provider.lastSuccessfulSnapshot.activeCPU;
+      stats.pendingCPU += provider.lastSuccessfulSnapshot.pendingCPU;
+      stats.availableCPU += provider.lastSuccessfulSnapshot.availableCPU;
 
-  const stats = {
-    activeProviderCount: filteredProviders.length,
-    activeCPU: filteredProviders.map(x => x.lastSuccessfulSnapshot.activeCPU).reduce((a, b) => a + b, 0),
-    activeGPU: filteredProviders.map(x => x.lastSuccessfulSnapshot.activeGPU).reduce((a, b) => a + b, 0),
-    activeMemory: filteredProviders.map(x => x.lastSuccessfulSnapshot.activeMemory).reduce((a, b) => a + b, 0),
-    activeStorage: filteredProviders
-      .map(x => x.lastSuccessfulSnapshot.activeEphemeralStorage + x.lastSuccessfulSnapshot.activePersistentStorage)
-      .reduce((a, b) => a + b, 0),
-    pendingCPU: filteredProviders.map(x => x.lastSuccessfulSnapshot.pendingCPU).reduce((a, b) => a + b, 0),
-    pendingGPU: filteredProviders.map(x => x.lastSuccessfulSnapshot.pendingGPU).reduce((a, b) => a + b, 0),
-    pendingMemory: filteredProviders.map(x => x.lastSuccessfulSnapshot.pendingMemory).reduce((a, b) => a + b, 0),
-    pendingStorage: filteredProviders
-      .map(x => x.lastSuccessfulSnapshot.pendingEphemeralStorage + x.lastSuccessfulSnapshot.pendingPersistentStorage)
-      .reduce((a, b) => a + b, 0),
-    availableCPU: filteredProviders.map(x => x.lastSuccessfulSnapshot.availableCPU).reduce((a, b) => a + b, 0),
-    availableGPU: filteredProviders.map(x => x.lastSuccessfulSnapshot.availableGPU).reduce((a, b) => a + b, 0),
-    availableMemory: filteredProviders.map(x => x.lastSuccessfulSnapshot.availableMemory).reduce((a, b) => a + b, 0),
-    availableStorage: filteredProviders
-      .map(x => x.lastSuccessfulSnapshot.availableEphemeralStorage + x.lastSuccessfulSnapshot.availablePersistentStorage)
-      .reduce((a, b) => a + b, 0)
-  };
+      stats.activeGPU += provider.lastSuccessfulSnapshot.activeGPU;
+      stats.pendingGPU += provider.lastSuccessfulSnapshot.pendingGPU;
+      stats.availableGPU += provider.lastSuccessfulSnapshot.availableGPU;
+
+      stats.activeMemory += provider.lastSuccessfulSnapshot.activeMemory;
+      stats.pendingMemory += provider.lastSuccessfulSnapshot.pendingMemory;
+      stats.availableMemory += provider.lastSuccessfulSnapshot.availableMemory;
+
+      stats.activeEphemeralStorage += provider.lastSuccessfulSnapshot.activeEphemeralStorage;
+      stats.pendingEphemeralStorage += provider.lastSuccessfulSnapshot.pendingEphemeralStorage;
+      stats.availableEphemeralStorage += provider.lastSuccessfulSnapshot.availableEphemeralStorage;
+
+      stats.activePersistentStorage += provider.lastSuccessfulSnapshot.activePersistentStorage;
+      stats.pendingPersistentStorage += provider.lastSuccessfulSnapshot.pendingPersistentStorage;
+      stats.availablePersistentStorage += provider.lastSuccessfulSnapshot.availablePersistentStorage;
+
+      return all;
+    },
+    {
+      activeCPU: 0,
+      pendingCPU: 0,
+      availableCPU: 0,
+      activeGPU: 0,
+      pendingGPU: 0,
+      availableGPU: 0,
+      activeMemory: 0,
+      pendingMemory: 0,
+      availableMemory: 0,
+      activeStorage: 0,
+      pendingStorage: 0,
+      availableStorage: 0,
+      activeEphemeralStorage: 0,
+      pendingEphemeralStorage: 0,
+      availableEphemeralStorage: 0,
+      activePersistentStorage: 0,
+      pendingPersistentStorage: 0,
+      availablePersistentStorage: 0
+    }
+  );
 
   return {
+    activeProviderCount: filteredProviders.length,
+    activeStorage: stats.activeEphemeralStorage + stats.activePersistentStorage,
+    pendingStorage: stats.pendingEphemeralStorage + stats.pendingPersistentStorage,
+    availableStorage: stats.availableEphemeralStorage + stats.availablePersistentStorage,
     ...stats,
     totalCPU: stats.activeCPU + stats.pendingCPU + stats.availableCPU,
     totalGPU: stats.activeGPU + stats.pendingGPU + stats.availableGPU,
@@ -96,10 +124,8 @@ export const getProviderList = async () => {
   });
 
   const distinctProviders = providersWithAttributesAndAuditors.filter((value, index, self) => self.map(x => x.hostUri).lastIndexOf(value.hostUri) === index);
-  const providerAttributeSchemaQuery = getProviderAttributesSchema();
-  const auditorsQuery = getAuditors();
 
-  const [auditors, providerAttributeSchema] = await Promise.all([auditorsQuery, providerAttributeSchemaQuery]);
+  const [auditors, providerAttributeSchema] = await Promise.all([getAuditors(), getProviderAttributesSchema()]);
 
   return distinctProviders.map(x => {
     const lastSuccessfulSnapshot = providerWithNodes.find(p => p.owner === x.owner)?.lastSuccessfulSnapshot;
@@ -151,10 +177,7 @@ export const getProviderDetail = async (address: string): Promise<ProviderDetail
       })
     : null;
 
-  const providerAttributeSchemaQuery = getProviderAttributesSchema();
-  const auditorsQuery = getAuditors();
-
-  const [auditors, providerAttributeSchema] = await Promise.all([auditorsQuery, providerAttributeSchemaQuery]);
+  const [auditors, providerAttributeSchema] = await Promise.all([getAuditors(), getProviderAttributesSchema()]);
 
   return {
     ...mapProviderToList(provider, providerAttributeSchema, auditors, lastSuccessfulSnapshot),

apps/api/src/types/provider.ts

@@ -24,18 +24,30 @@ export interface ProviderList {
   lastOnlineDate: Date;
   isAudited: boolean;
   gpuModels: { vendor: string; model: string; ram: string; interface: string }[];
+  stats: {
+    cpu: StatsItem;
+    gpu: StatsItem;
+    memory: StatsItem;
+    storage: {
+      ephemeral: StatsItem;
+      persistent: StatsItem;
+    },
+  };
+  /** @deprecated use `stats` instead */
   activeStats: {
     cpu: number;
     gpu: number;
     memory: number;
     storage: number;
   };
+  /** @deprecated use `stats` instead */
   pendingStats: {
     cpu: number;
     gpu: number;
     memory: number;
     storage: number;
   };
+  /** @deprecated use `stats` instead */
   availableStats: {
     cpu: number;
     gpu: number;
@@ -130,3 +142,9 @@ export type Auditor = {
   address: string;
   website: string;
 };
+
+export interface StatsItem {
+  active: number;
+  available: number;
+  pending: number;
+}

apps/api/src/utils/map/provider.ts

@@ -1,7 +1,7 @@
 import { Provider, ProviderSnapshot, ProviderSnapshotNode } from "@akashnetwork/database/dbSchemas/akash";
 import semver from "semver";
 
-import { Auditor, ProviderAttributesSchema, ProviderList } from "@src/types/provider";
+import { Auditor, ProviderAttributesSchema, ProviderList, StatsItem } from "@src/types/provider";
 import { createFilterUnique } from "../array/array";
 
 export const mapProviderToList = (
@@ -10,9 +10,18 @@ export const mapProviderToList = (
   auditors: Array<Auditor>,
   lastSuccessfulSnapshot?: ProviderSnapshot
 ): ProviderList => {
-  const isValidVersion = provider.cosmosSdkVersion ? semver.gte(provider.cosmosSdkVersion, "v0.45.9") : false;
+  const isValidSdkVersion = provider.cosmosSdkVersion ? semver.gte(provider.cosmosSdkVersion, "v0.45.9") : false;
   const name = provider.isOnline ? new URL(provider.hostUri).hostname : null;
   const gpuModels = getDistinctGpuModelsFromNodes(lastSuccessfulSnapshot?.nodes || []);
+  const stats: ProviderList['stats'] = {
+    cpu: buildStatsItem('CPU', lastSuccessfulSnapshot, isValidSdkVersion),
+    gpu: buildStatsItem('GPU', lastSuccessfulSnapshot, isValidSdkVersion),
+    memory: buildStatsItem('Memory', lastSuccessfulSnapshot, isValidSdkVersion),
+    storage: {
+      ephemeral: buildStatsItem('EphemeralStorage', lastSuccessfulSnapshot, isValidSdkVersion),
+      persistent: buildStatsItem('PersistentStorage', lastSuccessfulSnapshot, isValidSdkVersion),
+    },
+  };
 
   return {
     owner: provider.owner,
@@ -32,29 +41,15 @@ export const mapProviderToList = (
     ipCountryCode: provider.ipCountryCode,
     ipLat: provider.ipLat,
     ipLon: provider.ipLon,
-    activeStats: {
-      cpu: lastSuccessfulSnapshot?.activeCPU,
-      gpu: lastSuccessfulSnapshot?.activeGPU,
-      memory: lastSuccessfulSnapshot?.activeMemory,
-      storage: lastSuccessfulSnapshot?.activeEphemeralStorage + lastSuccessfulSnapshot?.activePersistentStorage
-    },
-    pendingStats: {
-      cpu: isValidVersion ? lastSuccessfulSnapshot?.pendingCPU : 0,
-      gpu: isValidVersion ? lastSuccessfulSnapshot?.pendingGPU : 0,
-      memory: isValidVersion ? lastSuccessfulSnapshot?.pendingMemory : 0,
-      storage: isValidVersion ? lastSuccessfulSnapshot?.pendingEphemeralStorage + lastSuccessfulSnapshot?.pendingPersistentStorage : 0
-    },
-    availableStats: {
-      cpu: isValidVersion ? lastSuccessfulSnapshot?.availableCPU : 0,
-      gpu: isValidVersion ? lastSuccessfulSnapshot?.availableGPU : 0,
-      memory: isValidVersion ? lastSuccessfulSnapshot?.availableMemory : 0,
-      storage: isValidVersion ? lastSuccessfulSnapshot?.availableEphemeralStorage + lastSuccessfulSnapshot?.availablePersistentStorage : 0
-    },
+    stats,
+    activeStats: buildLegacyStatsItem(stats, 'active'),
+    pendingStats: buildLegacyStatsItem(stats, 'pending'),
+    availableStats: buildLegacyStatsItem(stats, 'pending'),
     gpuModels: gpuModels,
     uptime1d: provider.uptime1d,
     uptime7d: provider.uptime7d,
     uptime30d: provider.uptime30d,
-    isValidVersion,
+    isValidVersion: isValidSdkVersion,
     isOnline: provider.isOnline,
     lastOnlineDate: lastSuccessfulSnapshot?.checkDate,
     isAudited: provider.providerAttributeSignatures.some(a => auditors.some(y => y.address === a.auditor)),
@@ -93,6 +88,32 @@ export const mapProviderToList = (
   } as ProviderList;
 };
 
+type StatsEntry = 'CPU' | 'GPU' | 'Memory' | 'PersistentStorage' | 'EphemeralStorage';
+function buildStatsItem<T extends StatsEntry>(suffix: T, snapshot: ProviderSnapshot | undefined | null, isValidSdkVersion: boolean): StatsItem {
+  if (!isValidSdkVersion) {
+    return {
+      active: snapshot?.[`active${suffix}`] || 0,
+      available: 0,
+      pending: 0,
+    };
+  }
+
+  return {
+    active: snapshot?.[`active${suffix}`] || 0,
+    available: snapshot?.[`available${suffix}`] || 0,
+    pending: snapshot?.[`pending${suffix}`] || 0,
+  };
+}
+
+function buildLegacyStatsItem(stats: ProviderList['stats'], type: keyof StatsItem) {
+  return {
+    cpu: stats.cpu[type],
+    gpu: stats.gpu[type],
+    memory: stats.memory[type],
+    storage: stats.storage.ephemeral[type] + stats.storage.persistent[type],
+  };
+}
+
 function getDistinctGpuModelsFromNodes(nodes: ProviderSnapshotNode[]) {
   const gpuModels = nodes.flatMap(x => x.gpus).map(x => ({ vendor: x.vendor, model: x.name, ram: x.memorySize, interface: x.interface }));
   const distinctGpuModels = gpuModels.filter(
@@ -102,8 +123,8 @@ function getDistinctGpuModelsFromNodes(nodes: ProviderSnapshotNode[]) {
   return distinctGpuModels;
 }
 
-export const getProviderAttributeValue = (
-  key: keyof ProviderAttributesSchema,
+export const getProviderAttributeValue = <TKey extends keyof ProviderAttributesSchema>(
+  key: TKey,
   provider: Provider,
   providerAttributeSchema: ProviderAttributesSchema
 ): string | string[] | boolean | number | null => {