From 24817a870e9e593fa3030f8531ab5e2db05b69ed Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Fri, 7 Feb 2025 12:28:28 -0800 Subject: [PATCH 01/11] common,cli,agent: remove multi-network-mode --- packages/indexer-agent/README.md | 6 - .../indexer-agent/src/__tests__/indexer.ts | 8 +- packages/indexer-agent/src/agent.ts | 729 ++++++------------ .../src/commands/common-options.ts | 8 - .../src/commands/start-multi-network.ts | 18 - packages/indexer-agent/src/commands/start.ts | 51 +- packages/indexer-agent/src/index.ts | 27 +- packages/indexer-agent/src/syncing-server.ts | 11 +- packages/indexer-agent/src/types.ts | 4 +- .../indexer-cli/src/__tests__/cli.test.ts | 4 +- .../src/__tests__/indexer/actions.test.ts | 4 +- .../src/__tests__/indexer/cost.test.ts | 3 +- .../src/__tests__/indexer/rules.test.ts | 4 +- packages/indexer-cli/src/__tests__/util.ts | 18 +- packages/indexer-common/src/index.ts | 1 - .../__tests__/helpers.test.ts | 49 -- .../src/indexer-management/__tests__/util.ts | 8 +- .../src/indexer-management/actions.ts | 130 ++-- .../src/indexer-management/client.ts | 16 +- .../src/indexer-management/monitor.ts | 88 --- .../indexer-management/resolvers/actions.ts | 44 +- .../resolvers/allocations.ts | 124 ++- .../resolvers/cost-models.ts | 7 +- .../resolvers/indexer-status.ts | 61 +- .../src/indexer-management/resolvers/utils.ts | 26 - .../src/indexer-management/types.ts | 14 - packages/indexer-common/src/multi-networks.ts | 111 --- .../indexer-common/src/parsers/test-utils.ts | 26 +- packages/indexer-common/src/types.ts | 19 - 29 files changed, 472 insertions(+), 1147 deletions(-) delete mode 100644 packages/indexer-agent/src/commands/start-multi-network.ts delete mode 100644 packages/indexer-common/src/indexer-management/resolvers/utils.ts delete mode 100644 packages/indexer-common/src/multi-networks.ts diff --git a/packages/indexer-agent/README.md b/packages/indexer-agent/README.md index a0173135d..af735993a 100644 --- a/packages/indexer-agent/README.md +++ b/packages/indexer-agent/README.md @@ -49,9 +49,6 @@ Indexer Infrastructure --graph-node-admin-endpoint Graph Node endpoint for applying and updating subgraph deployments [string] [required] - --enable-auto-migration-support Auto migrate allocations from L1 to L2 - (multi-network mode must be enabled) - [boolean] [default: false] --public-indexer-url Indexer endpoint for receiving requests from the network [string] [required] --indexer-geo-coordinates Coordinates describing the Indexer's @@ -161,9 +158,6 @@ Indexer Infrastructure etc. [string] [required] --graph-node-admin-endpoint Graph Node endpoint for applying and updating subgraph deployments [string] [required] - --enable-auto-migration-support Auto migrate allocations from L1 to L2 - (multi-network mode must be enabled) - [boolean] [default: false] Postgres --postgres-host Postgres host [string] [required] diff --git a/packages/indexer-agent/src/__tests__/indexer.ts b/packages/indexer-agent/src/__tests__/indexer.ts index f8c1567de..419c90556 100644 --- a/packages/indexer-agent/src/__tests__/indexer.ts +++ b/packages/indexer-agent/src/__tests__/indexer.ts @@ -18,7 +18,6 @@ import { specification, QueryFeeModels, defineQueryFeeModels, - MultiNetworks, loadTestYamlConfig, } from '@graphprotocol/indexer-common' import { BigNumber } from 'ethers' @@ -152,11 +151,6 @@ const setup = async () => { metrics, ) - const multiNetworks = new MultiNetworks( - [network], - (n: Network) => n.specification.networkIdentifier, - ) - indexerManagementClient = await createIndexerManagementClient({ models, graphNode, @@ -167,7 +161,7 @@ const setup = async () => { parallelAllocations: 1, }, }, - multiNetworks, + network, }) operator = new Operator(logger, indexerManagementClient, networkSpecification) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 82e14c57e..790143429 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -8,7 +8,6 @@ import { timer, } from '@graphprotocol/common-ts' import { - ActivationCriteria, ActionStatus, Allocation, AllocationManagementMode, @@ -30,11 +29,6 @@ import { GraphNode, Operator, validateProviderNetworkIdentifier, - MultiNetworks, - NetworkMapped, - TransferredSubgraphDeployment, - networkIsL2, - networkIsL1, DeploymentManagementMode, SubgraphStatus, sequentialTimerMap, @@ -43,12 +37,8 @@ import { import PQueue from 'p-queue' import pMap from 'p-map' import pFilter from 'p-filter' -import mapValues from 'lodash.mapvalues' -import zip from 'lodash.zip' import { AgentConfigs, NetworkAndOperator } from './types' -type ActionReconciliationContext = [AllocationDecision[], number, number] - const deploymentInList = ( list: SubgraphDeploymentID[], deployment: SubgraphDeploymentID, @@ -63,7 +53,7 @@ const deploymentRuleInList = ( rule => rule.identifierType == SubgraphIdentifierType.DEPLOYMENT && new SubgraphDeploymentID(rule.identifier).toString() == - deployment.toString(), + deployment.toString(), ) !== undefined const uniqueDeploymentsOnly = ( @@ -128,65 +118,11 @@ export const convertSubgraphBasedRulesToDeploymentBased = ( return rules } -// Extracts the network identifier from a pair of matching Network and Operator objects. -function networkAndOperatorIdentity({ - network, - operator, -}: NetworkAndOperator): string { - const networkId = network.specification.networkIdentifier - const operatorId = operator.specification.networkIdentifier - if (networkId !== operatorId) { - throw new Error( - `Network and Operator pairs have different network identifiers: ${networkId} != ${operatorId}`, - ) - } - return networkId -} - -// Helper function to produce a `MultiNetworks` while validating its -// inputs. -function createMultiNetworks( - networks: Network[], - operators: Operator[], -): MultiNetworks { - // Validate that Networks and Operator arrays have even lengths and - // contain unique, matching network identifiers. - const visited = new Set() - const validInputs = - networks.length === operators.length && - networks.every((network, index) => { - const sameIdentifier = - network.specification.networkIdentifier === - operators[index].specification.networkIdentifier - if (!sameIdentifier) { - return false - } - if (visited.has(network.specification.networkIdentifier)) { - return false - } - visited.add(network.specification.networkIdentifier) - return true - }) - - if (!validInputs) { - throw new Error( - 'Invalid Networks and Operator pairs used in Agent initialization', - ) - } - // Note on undefineds: `lodash.zip` can return `undefined` if array lengths are - // uneven, but we have just checked that. - const networksAndOperators = zip(networks, operators).map(pair => { - const [network, operator] = pair - return { network: network!, operator: operator! } - }) - return new MultiNetworks(networksAndOperators, networkAndOperatorIdentity) -} - export class Agent { logger: Logger metrics: Metrics graphNode: GraphNode - multiNetworks: MultiNetworks + networkAndOperator: NetworkAndOperator indexerManagement: IndexerManagementClient offchainSubgraphs: SubgraphDeploymentID[] autoMigrationSupport: boolean @@ -198,10 +134,10 @@ export class Agent { this.metrics = configs.metrics this.graphNode = configs.graphNode this.indexerManagement = configs.indexerManagement - this.multiNetworks = createMultiNetworks( - configs.networks, - configs.operators, - ) + this.networkAndOperator = { + network: configs.network, + operator: configs.operator, + } this.offchainSubgraphs = configs.offchainSubgraphs this.autoMigrationSupport = !!configs.autoMigrationSupport this.deploymentManagement = configs.deploymentManagement @@ -228,23 +164,20 @@ export class Agent { // * Ensure NetworkSubgraph is indexing // * Register the Indexer in the Network // -------------------------------------------------------------------------------- - await this.multiNetworks.map( - async ({ network, operator }: NetworkAndOperator) => { - try { - await operator.ensureGlobalIndexingRule() - await this.ensureAllSubgraphsIndexing(network) - await network.register() - } catch (err) { - this.logger.critical( - `Failed to prepare indexer for ${network.specification.networkIdentifier}`, - { - error: err.message, - }, - ) - process.exit(1) - } - }, - ) + const { network, operator }: NetworkAndOperator = this.networkAndOperator + try { + await operator.ensureGlobalIndexingRule() + await this.ensureAllSubgraphsIndexing(network) + await network.register() + } catch (err) { + this.logger.critical( + `Failed to prepare indexer for ${network.specification.networkIdentifier}`, + { + error: err.message, + }, + ) + process.exit(1) + } this.reconciliationLoop() return this @@ -254,67 +187,65 @@ export class Agent { const requestIntervalSmall = this.pollingInterval const requestIntervalLarge = this.pollingInterval * 5 const logger = this.logger.child({ component: 'ReconciliationLoop' }) - const currentEpochNumber: Eventual> = - sequentialTimerMap( - { logger, milliseconds: requestIntervalLarge }, - async () => - await this.multiNetworks.map(({ network }) => { - logger.trace('Fetching current epoch number', { - protocolNetwork: network.specification.networkIdentifier, - }) - return network.networkMonitor.currentEpochNumber() - }), - { - onError: error => - logger.warn(`Failed to fetch current epoch`, { error }), - }, - ) + const currentEpochNumber: Eventual = sequentialTimerMap( + { logger, milliseconds: requestIntervalLarge }, + async () => { + const { network } = this.networkAndOperator + logger.trace('Fetching current epoch number', { + protocolNetwork: network.specification.networkIdentifier, + }) + return await network.networkMonitor.currentEpochNumber() + }, + { + onError: error => + logger.warn(`Failed to fetch current epoch`, { error }), + }, + ) - const maxAllocationEpochs: Eventual> = - sequentialTimerMap( - { logger, milliseconds: requestIntervalLarge }, - () => - this.multiNetworks.map(({ network }) => { - logger.trace('Fetching max allocation epochs', { - protocolNetwork: network.specification.networkIdentifier, - }) - return network.contracts.staking.maxAllocationEpochs() - }), - { - onError: error => - logger.warn(`Failed to fetch max allocation epochs`, { error }), - }, - ) + const { network } = this.networkAndOperator - const indexingRules: Eventual> = + const maxAllocationEpochs: Eventual = sequentialTimerMap( + { logger, milliseconds: requestIntervalLarge }, + async () => { + logger.trace('Fetching max allocation epochs', { + protocolNetwork: network.specification.networkIdentifier, + }) + return network.contracts.staking.maxAllocationEpochs() + }, + { + onError: error => + logger.warn(`Failed to fetch max allocation epochs`, { error }), + }, + ) + + const indexingRules: Eventual = sequentialTimerMap( { logger, milliseconds: requestIntervalSmall }, async () => { - return this.multiNetworks.map(async ({ network, operator }) => { - logger.trace('Fetching indexing rules', { - protocolNetwork: network.specification.networkIdentifier, - }) - let rules = await operator.indexingRules(true) - const subgraphRuleIds = rules - .filter( - rule => rule.identifierType == SubgraphIdentifierType.SUBGRAPH, - ) - .map(rule => rule.identifier!) - const subgraphsMatchingRules = - await network.networkMonitor.subgraphs(subgraphRuleIds) - if (subgraphsMatchingRules.length >= 1) { - const epochLength = - await network.contracts.epochManager.epochLength() - const blockPeriod = 15 - const bufferPeriod = epochLength.toNumber() * blockPeriod * 100 // 100 epochs - rules = convertSubgraphBasedRulesToDeploymentBased( - rules, - subgraphsMatchingRules, - bufferPeriod, - ) - } - return rules + const { network, operator } = this.networkAndOperator + logger.trace('Fetching indexing rules', { + protocolNetwork: network.specification.networkIdentifier, }) + let rules = await operator.indexingRules(true) + const subgraphRuleIds = rules + .filter( + rule => rule.identifierType == SubgraphIdentifierType.SUBGRAPH, + ) + .map(rule => rule.identifier!) + const subgraphsMatchingRules = + await network.networkMonitor.subgraphs(subgraphRuleIds) + if (subgraphsMatchingRules.length >= 1) { + const epochLength = + await network.contracts.epochManager.epochLength() + const blockPeriod = 15 + const bufferPeriod = epochLength.toNumber() * blockPeriod * 100 // 100 epochs + rules = convertSubgraphBasedRulesToDeploymentBased( + rules, + subgraphsMatchingRules, + bufferPeriod, + ) + } + return rules }, { onError: error => @@ -329,8 +260,12 @@ export class Agent { sequentialTimerMap( { logger, milliseconds: requestIntervalLarge }, async () => { - if (this.deploymentManagement === DeploymentManagementMode.AUTO) { - logger.debug('Fetching active deployments') + const { network } = this.networkAndOperator + if ( + this.deploymentManagement === DeploymentManagementMode.AUTO || + network.networkMonitor.poiDisputeMonitoringEnabled() + ) { + logger.trace('Fetching active deployments') const assignments = await this.graphNode.subgraphDeploymentsAssignments( SubgraphStatus.ACTIVE, @@ -351,16 +286,16 @@ export class Agent { }, ) - const networkDeployments: Eventual> = + const networkDeployments: Eventual = sequentialTimerMap( { logger, milliseconds: requestIntervalSmall }, - async () => - await this.multiNetworks.map(({ network }) => { - logger.trace('Fetching network deployments', { - protocolNetwork: network.specification.networkIdentifier, - }) - return network.networkMonitor.subgraphDeployments() - }), + async () => { + const { network } = this.networkAndOperator + logger.trace('Fetching network deployments', { + protocolNetwork: network.specification.networkIdentifier, + }) + return network.networkMonitor.subgraphDeployments() + }, { onError: error => logger.warn( @@ -370,173 +305,26 @@ export class Agent { }, ) - const eligibleTransferDeployments: Eventual< - NetworkMapped - > = sequentialTimerMap( - { logger, milliseconds: requestIntervalLarge }, - async () => { - // Return early if the auto migration feature is disabled. - if (!this.autoMigrationSupport) { - logger.trace( - 'Auto Migration feature is disabled, skipping querying transferred subgraphs', - ) - return this.multiNetworks.map(async () => []) - } - - const statuses = await this.graphNode.indexingStatus([]) - return this.multiNetworks.map(async ({ network }) => { - const protocolNetwork = network.specification.networkIdentifier - logger.trace('Fetching deployments eligible for L2 transfer', { - protocolNetwork, - }) - const transfers = - await network.networkMonitor.transferredDeployments() - logger.trace( - `Found ${transfers.length} transferred subgraphs in the network`, - { protocolNetwork }, - ) - return transfers - .map(transfer => { - const status = statuses.find( - status => - status.subgraphDeployment.ipfsHash == transfer.ipfsHash, - ) - if (status) { - transfer.ready = status.synced && status.health == 'healthy' - } - return transfer - }) - .filter(transfer => transfer.ready == true) - }) - }, - { - onError: error => - logger.warn( - `Failed to obtain transferred deployments, trying again later`, - { error }, - ), - }, - ) - - // While in the L1 -> L2 transfer period this will be an intermediate value - // with the final value including transfer considerations - const intermediateNetworkDeploymentAllocationDecisions: Eventual< - NetworkMapped - > = join({ - networkDeployments, - indexingRules, - }).tryMap( - ({ indexingRules, networkDeployments }) => { - return mapValues( - this.multiNetworks.zip(indexingRules, networkDeployments), - ([indexingRules, networkDeployments]: [ - IndexingRuleAttributes[], - SubgraphDeployment[], - ]) => { - // Identify subgraph deployments on the network that are worth picking up; - // these may overlap with the ones we're already indexing - logger.trace('Evaluating which deployments are worth allocating to') - return indexingRules.length === 0 - ? [] - : evaluateDeployments(logger, networkDeployments, indexingRules) - }, - ) - }, - { - onError: error => - logger.warn(`Failed to evaluate deployments, trying again later`, { - error, - }), - }, - ) - - // Update targetDeployments and networkDeplomentAllocationDecisions using transferredSubgraphDeployments data - // This will be somewhat custom and will likely be yanked out later after the transfer stage is complete - // Cases: - // - L1 subgraph that had the transfer started: keep synced and allocated to for at least one week - // post transfer. - // - L2 subgraph that has been transferred: - // - if already synced, allocate to it immediately using default allocation amount - // - if not synced, no changes - const networkDeploymentAllocationDecisions: Eventual< - NetworkMapped - > = join({ - intermediateNetworkDeploymentAllocationDecisions, - eligibleTransferDeployments, - }).tryMap( - ({ - intermediateNetworkDeploymentAllocationDecisions, - eligibleTransferDeployments, - }) => - mapValues( - this.multiNetworks.zip( - intermediateNetworkDeploymentAllocationDecisions, - eligibleTransferDeployments, - ), - ([allocationDecisions, eligibleTransferDeployments]: [ - AllocationDecision[], - TransferredSubgraphDeployment[], - ]) => { - logger.debug( - `Found ${eligibleTransferDeployments.length} deployments eligible for transfer`, - { eligibleTransferDeployments }, - ) - const oneWeekAgo = Math.floor(Date.now() / 1_000) - 86_400 * 7 - return allocationDecisions.map(decision => { - const matchingTransfer = eligibleTransferDeployments.find( - deployment => - deployment.ipfsHash == decision.deployment.ipfsHash && - deployment.startedTransferToL2At.toNumber() > oneWeekAgo, - ) - if (matchingTransfer) { - logger.debug('Found a matching subgraph transfer', { - matchingTransfer, - }) - // L1 deployments being transferred need to be supported for one week post transfer - // to ensure continued support. - if (networkIsL1(matchingTransfer.protocolNetwork)) { - decision.toAllocate = true - decision.ruleMatch.activationCriteria = - ActivationCriteria.L2_TRANSFER_SUPPORT - logger.debug( - `Allocating towards L1 subgraph deployment to support its transfer`, - { - subgraphDeployment: matchingTransfer, - allocationDecision: decision, - }, - ) - } - // L2 Deployments - if ( - networkIsL2(matchingTransfer.protocolNetwork) && - !!matchingTransfer.transferredToL2 - ) { - decision.toAllocate = true - decision.ruleMatch.activationCriteria = - ActivationCriteria.L2_TRANSFER_SUPPORT - logger.debug( - `Allocating towards transferred L2 subgraph deployment`, - { - subgraphDeployment: matchingTransfer, - allocationDecision: decision, - }, - ) - } - } - return decision - }) - }, - ), - { - onError: error => - logger.warn( - `Failed to merge L2 transfer decisions, trying again later`, - { + const networkDeploymentAllocationDecisions: Eventual = + join({ + networkDeployments, + indexingRules, + }).tryMap( + ({ indexingRules, networkDeployments }) => { + // Identify subgraph deployments on the network that are worth picking up; + // these may overlap with the ones we're already indexing + logger.trace('Evaluating which deployments are worth allocating to') + return indexingRules.length === 0 + ? [] + : evaluateDeployments(logger, networkDeployments, indexingRules) + }, + { + onError: error => + logger.warn(`Failed to evaluate deployments, trying again later`, { error, - }, - ), - }, - ) + }), + }, + ) // let targetDeployments be an union of targetAllocations // and offchain subgraphs. @@ -546,8 +334,12 @@ export class Agent { }).tryMap( async ({ indexingRules, networkDeploymentAllocationDecisions }) => { logger.trace('Resolving target deployments') - const targetDeploymentIDs: Set = - consolidateAllocationDecisions(networkDeploymentAllocationDecisions) + const targetDeploymentIDs: Set = new Set( + Object.values(networkDeploymentAllocationDecisions) + .flat() + .filter(decision => decision.toAllocate === true) + .map(decision => decision.deployment), + ) // Add offchain subgraphs to the deployment list from rules Object.values(indexingRules) @@ -572,23 +364,22 @@ export class Agent { }, ) - const activeAllocations: Eventual> = - sequentialTimerMap( - { logger, milliseconds: requestIntervalSmall }, - () => - this.multiNetworks.map(({ network }) => { - logger.trace('Fetching active allocations', { - protocolNetwork: network.specification.networkIdentifier, - }) - return network.networkMonitor.allocations(AllocationStatus.ACTIVE) - }), - { - onError: () => - logger.warn( - `Failed to obtain active allocations, trying again later`, - ), - }, - ) + const activeAllocations: Eventual = sequentialTimerMap( + { logger, milliseconds: requestIntervalSmall }, + async () => { + const { network } = this.networkAndOperator + logger.trace('Fetching active allocations', { + protocolNetwork: network.specification.networkIdentifier, + }) + return network.networkMonitor.allocations(AllocationStatus.ACTIVE) + }, + { + onError: () => + logger.warn( + `Failed to obtain active allocations, trying again later`, + ), + }, + ) // `activeAllocations` is used to trigger this Eventual, but not really needed // inside. @@ -598,19 +389,12 @@ export class Agent { }).tryMap( // eslint-disable-next-line @typescript-eslint/no-unused-vars async ({ activeAllocations: _, currentEpochNumber }) => { - const allocationsByNetwork = await this.multiNetworks.mapNetworkMapped( - currentEpochNumber, - async ({ network }, epochNumber): Promise => { - logger.trace('Fetching recently closed allocations', { - protocolNetwork: network.specification.networkIdentifier, - currentEpochNumber, - }) - return network.networkMonitor.recentlyClosedAllocations( - epochNumber, - 1, - ) - }, - ) + const { network } = this.networkAndOperator + const allocationsByNetwork = + await network.networkMonitor.recentlyClosedAllocations( + currentEpochNumber, + 1, + ) return Object.values(allocationsByNetwork).flat() }, { @@ -621,26 +405,22 @@ export class Agent { }, ) - const disputableAllocations: Eventual> = join({ + const disputableAllocations: Eventual = join({ currentEpochNumber, activeDeployments, }).tryMap( - async ({ currentEpochNumber, activeDeployments }) => - this.multiNetworks.mapNetworkMapped( + async ({ currentEpochNumber, activeDeployments }) => { + const { network } = this.networkAndOperator + logger.trace('Fetching disputable allocations', { + protocolNetwork: network.specification.networkIdentifier, currentEpochNumber, - ({ network }: NetworkAndOperator, currentEpochNumber: number) => { - logger.trace('Fetching disputable allocations', { - protocolNetwork: network.specification.networkIdentifier, - currentEpochNumber, - }) - return network.networkMonitor.disputableAllocations( - currentEpochNumber, - activeDeployments, - 0, - ) - }, - ), - + }) + return network.networkMonitor.disputableAllocations( + currentEpochNumber, + activeDeployments, + 0, + ) + }, { onError: () => logger.warn( @@ -675,30 +455,17 @@ export class Agent { }) try { - const disputableEpochs = await this.multiNetworks.mapNetworkMapped( - currentEpochNumber, - async ( - { network }: NetworkAndOperator, - currentEpochNumber: number, - ) => - currentEpochNumber - - network.specification.indexerOptions.poiDisputableEpochs, - ) + const { network, operator } = this.networkAndOperator + const disputableEpochs = + currentEpochNumber - + network.specification.indexerOptions.poiDisputableEpochs // Find disputable allocations - await this.multiNetworks.mapNetworkMapped( - this.multiNetworks.zip(disputableEpochs, disputableAllocations), - async ( - { network, operator }: NetworkAndOperator, - [disputableEpoch, disputableAllocations]: [number, Allocation[]], - ): Promise => { - await this.identifyPotentialDisputes( - disputableAllocations, - disputableEpoch, - operator, - network, - ) - }, + await this.identifyPotentialDisputes( + disputableAllocations, + disputableEpochs, + operator, + network, ) } catch (err) { logger.warn(`Failed POI dispute monitoring`, { err }) @@ -859,7 +626,7 @@ export class Agent { let status = rewardsPool!.referencePOI == allocation.poi || - rewardsPool!.referencePreviousPOI == allocation.poi + rewardsPool!.referencePreviousPOI == allocation.poi ? 'valid' : 'potential' @@ -916,15 +683,14 @@ export class Agent { // Ensure the network subgraph deployment is _always_ indexed // ---------------------------------------------------------------------------------------- let indexingNetworkSubgraph = false - await this.multiNetworks.map(async ({ network }) => { - if (network.networkSubgraph.deployment) { - const networkDeploymentID = network.networkSubgraph.deployment.id - if (!deploymentInList(targetDeployments, networkDeploymentID)) { - targetDeployments.push(networkDeploymentID) - indexingNetworkSubgraph = true - } + const { network } = this.networkAndOperator + if (network.networkSubgraph.deployment) { + const networkDeploymentID = network.networkSubgraph.deployment.id + if (!deploymentInList(targetDeployments, networkDeploymentID)) { + targetDeployments.push(networkDeploymentID) + indexingNetworkSubgraph = true } - }) + } // ---------------------------------------------------------------------------------------- // Inspect Deployments and Networks @@ -1143,116 +909,95 @@ export class Agent { } async reconcileActions( - networkDeploymentAllocationDecisions: NetworkMapped, - epoch: NetworkMapped, - maxAllocationEpochs: NetworkMapped, + allocationDecisions: AllocationDecision[], + epoch: number, + maxAllocationEpochs: number, ): Promise { // -------------------------------------------------------------------------------- // Filter out networks set to `manual` allocation management mode, and ensure the // Network Subgraph is NEVER allocated towards // -------------------------------------------------------------------------------- - const validatedAllocationDecisions = - await this.multiNetworks.mapNetworkMapped( - networkDeploymentAllocationDecisions, - async ( - { network }: NetworkAndOperator, - allocationDecisions: AllocationDecision[], - ) => { - if ( - network.specification.indexerOptions.allocationManagementMode === - AllocationManagementMode.MANUAL - ) { - this.logger.trace( - `Skipping allocation reconciliation since AllocationManagementMode = 'manual'`, - { - protocolNetwork: network.specification.networkIdentifier, - targetDeployments: allocationDecisions - .filter(decision => decision.toAllocate) - .map(decision => decision.deployment.ipfsHash), - }, - ) - return [] as AllocationDecision[] - } - const networkSubgraphDeployment = network.networkSubgraph.deployment - if ( - networkSubgraphDeployment && - !network.specification.indexerOptions.allocateOnNetworkSubgraph - ) { - const networkSubgraphIndex = allocationDecisions.findIndex( - decision => - decision.deployment.bytes32 == - networkSubgraphDeployment.id.bytes32, - ) - if (networkSubgraphIndex >= 0) { - allocationDecisions[networkSubgraphIndex].toAllocate = false - } - } - return allocationDecisions + const { network, operator } = this.networkAndOperator + let validatedAllocationDecisions = allocationDecisions + + if ( + network.specification.indexerOptions.allocationManagementMode === + AllocationManagementMode.MANUAL + ) { + this.logger.trace( + `Skipping allocation reconciliation since AllocationManagementMode = 'manual'`, + { + protocolNetwork: network.specification.networkIdentifier, + targetDeployments: allocationDecisions + .filter(decision => decision.toAllocate) + .map(decision => decision.deployment.ipfsHash), }, ) + validatedAllocationDecisions = [] as AllocationDecision[] + } else { + const networkSubgraphDeployment = network.networkSubgraph.deployment + if ( + networkSubgraphDeployment && + !network.specification.indexerOptions.allocateOnNetworkSubgraph + ) { + const networkSubgraphIndex = allocationDecisions.findIndex( + decision => + decision.deployment.bytes32 == networkSubgraphDeployment.id.bytes32, + ) + if (networkSubgraphIndex >= 0) { + allocationDecisions[networkSubgraphIndex].toAllocate = false + } + } + } //---------------------------------------------------------------------------------------- // For every network, loop through all deployments and queue allocation actions if needed //---------------------------------------------------------------------------------------- - await this.multiNetworks.mapNetworkMapped( - this.multiNetworks.zip3( - validatedAllocationDecisions, - epoch, - maxAllocationEpochs, - ), - async ( - { network, operator }: NetworkAndOperator, - [ - allocationDecisions, - epoch, - maxAllocationEpochs, - ]: ActionReconciliationContext, - ) => { - // Do nothing if there are already approved actions in the queue awaiting execution - const approvedActions = await operator.fetchActions({ - status: ActionStatus.APPROVED, - protocolNetwork: network.specification.networkIdentifier, - }) - if (approvedActions.length > 0) { - this.logger.info( - `There are ${approvedActions.length} approved actions awaiting execution, will reconcile with the network once they are executed`, - { protocolNetwork: network.specification.networkIdentifier }, - ) - return - } - // Accuracy check: re-fetch allocations to ensure that we have a fresh state since the - // start of the reconciliation loop. This means we don't use the allocations coming from - // the Eventual input. - const activeAllocations: Allocation[] = - await network.networkMonitor.allocations(AllocationStatus.ACTIVE) + // Do nothing if there are already approved actions in the queue awaiting execution + const approvedActions = await operator.fetchActions({ + status: ActionStatus.APPROVED, + protocolNetwork: network.specification.networkIdentifier, + }) + if (approvedActions.length > 0) { + this.logger.info( + `There are ${approvedActions.length} approved actions awaiting execution, will reconcile with the network once they are executed`, + { protocolNetwork: network.specification.networkIdentifier }, + ) + return + } - this.logger.trace(`Reconcile allocation actions`, { - protocolNetwork: network.specification.networkIdentifier, - epoch, - maxAllocationEpochs, - targetDeployments: allocationDecisions - .filter(decision => decision.toAllocate) - .map(decision => decision.deployment.ipfsHash), - activeAllocations: activeAllocations.map(allocation => ({ - id: allocation.id, - deployment: allocation.subgraphDeployment.id.ipfsHash, - createdAtEpoch: allocation.createdAtEpoch, - })), - }) + // Accuracy check: re-fetch allocations to ensure that we have a fresh state since the + // start of the reconciliation loop. This means we don't use the allocations coming from + // the Eventual input. + const activeAllocations: Allocation[] = + await network.networkMonitor.allocations(AllocationStatus.ACTIVE) - return pMap(allocationDecisions, async decision => - this.reconcileDeploymentAllocationAction( - decision, - activeAllocations, - epoch, - maxAllocationEpochs, - network, - operator, - ), - ) - }, + this.logger.trace(`Reconcile allocation actions`, { + protocolNetwork: network.specification.networkIdentifier, + epoch, + maxAllocationEpochs, + targetDeployments: validatedAllocationDecisions + .filter(decision => decision.toAllocate) + .map(decision => decision.deployment.ipfsHash), + activeAllocations: activeAllocations.map(allocation => ({ + id: allocation.id, + deployment: allocation.subgraphDeployment.id.ipfsHash, + createdAtEpoch: allocation.createdAtEpoch, + })), + }) + + await pMap(validatedAllocationDecisions, async decision => + this.reconcileDeploymentAllocationAction( + decision, + activeAllocations, + epoch, + maxAllocationEpochs, + network, + operator, + ), ) + return } // TODO: After indexer-service deprecation: Move to be an initialization check inside Network.create() diff --git a/packages/indexer-agent/src/commands/common-options.ts b/packages/indexer-agent/src/commands/common-options.ts index 0ed8e67c7..e98e37d9b 100644 --- a/packages/indexer-agent/src/commands/common-options.ts +++ b/packages/indexer-agent/src/commands/common-options.ts @@ -120,14 +120,6 @@ export function injectCommonStartupOptions(argv: Argv): Argv { required: true, group: 'Indexer Infrastructure', }) - .option('enable-auto-migration-support', { - description: - 'Auto migrate allocations from L1 to L2 (multi-network mode must be enabled)', - type: 'boolean', - required: false, - default: false, - group: 'Indexer Infrastructure', - }) .option('deployment-management', { describe: 'Subgraph deployments management mode', required: false, diff --git a/packages/indexer-agent/src/commands/start-multi-network.ts b/packages/indexer-agent/src/commands/start-multi-network.ts deleted file mode 100644 index 491cd8189..000000000 --- a/packages/indexer-agent/src/commands/start-multi-network.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Argv } from 'yargs' -import { injectCommonStartupOptions } from './common-options' - -export const startMultiNetwork = { - command: 'start', - describe: 'Start the Agent in multiple Protocol Networks', - builder: (args: Argv): Argv => { - const updatedArgs = injectCommonStartupOptions(args) - return updatedArgs.option('network-specifications-directory', { - alias: 'dir', - description: 'Path to a directory containing network specification files', - type: 'string', - required: true, - }) - }, - // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any - handler: (_argv: any) => {}, -} diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index caee24b45..004e48077 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -18,7 +18,6 @@ import { GraphNode, indexerError, IndexerErrorCode, - MultiNetworks, Network, Operator, registerIndexerErrorMetrics, @@ -28,8 +27,6 @@ import { import { Agent } from '../agent' import { createSyncingServer } from '../syncing-server' import { injectCommonStartupOptions } from './common-options' -import pMap from 'p-map' -import { NetworkSpecification } from '@graphprotocol/indexer-common/dist/network-specification' import { BigNumber } from 'ethers' import { displayZodParsingError } from '@graphprotocol/indexer-common' import { readFileSync } from 'fs' @@ -49,7 +46,15 @@ export const start = { describe: 'Start the agent', builder: (args: Argv): Argv => { const updatedArgs = injectCommonStartupOptions(args) + return updatedArgs + .option('network-specifications-directory', { + alias: 'dir', + description: + 'Path to a directory containing network specification files', + type: 'string', + required: false, + }) .option('network-provider', { alias: 'ethereum', description: 'Ethereum node or provider URL', @@ -459,7 +464,7 @@ function loadFile(path: string | undefined): unknown | undefined { export async function run( argv: AgentOptions, - networkSpecifications: spec.NetworkSpecification[], + networkSpecification: spec.NetworkSpecification, logger: Logger, ): Promise { // -------------------------------------------------------------------------------- @@ -543,7 +548,7 @@ export async function run( queryInterface: sequelize.getQueryInterface(), logger, graphNodeAdminEndpoint: argv.graphNodeAdminEndpoint, - networkSpecifications, + networkSpecifications: networkSpecification, graphNode: graphNode, }, storage: new SequelizeStorage({ sequelize }), @@ -573,23 +578,21 @@ export async function run( // -------------------------------------------------------------------------------- // * Networks // -------------------------------------------------------------------------------- - logger.info('Connect to network/s', { - networks: networkSpecifications.map(spec => spec.networkIdentifier), + logger.info('Connect to network', { + networks: networkSpecification.networkIdentifier, }) - const networks: Network[] = await pMap( - networkSpecifications, - async (spec: NetworkSpecification) => - Network.create(logger, spec, queryFeeModels, graphNode, metrics), + const network = await Network.create( + logger, + networkSpecification, + queryFeeModels, + graphNode, + metrics, ) // -------------------------------------------------------------------------------- // * Indexer Management (GraphQL) Server // -------------------------------------------------------------------------------- - const multiNetworks = new MultiNetworks( - networks, - (n: Network) => n.specification.networkIdentifier, - ) const indexerManagementClient = await createIndexerManagementClient({ models: managementModels, @@ -602,7 +605,7 @@ export async function run( parallelAllocations: 1, }, }, - multiNetworks, + network, }) // -------------------------------------------------------------------------------- @@ -625,9 +628,7 @@ export async function run( await createSyncingServer({ logger, - networkSubgraphs: await multiNetworks.map( - async network => network.networkSubgraph, - ), + networkSubgraph: network.networkSubgraph, port: argv.syncingPort, }) logger.info(`Successfully launched syncing server`) @@ -635,10 +636,10 @@ export async function run( // -------------------------------------------------------------------------------- // * Operator // -------------------------------------------------------------------------------- - const operators: Operator[] = await pMap( - networkSpecifications, - async (spec: NetworkSpecification) => - new Operator(logger, indexerManagementClient, spec), + const operator = new Operator( + logger, + indexerManagementClient, + networkSpecification, ) // -------------------------------------------------------------------------------- @@ -648,9 +649,9 @@ export async function run( logger, metrics, graphNode, - operators, + operator, indexerManagement: indexerManagementClient, - networks, + network, deploymentManagement: argv.deploymentManagement, autoMigrationSupport: argv.enableAutoMigrationSupport, offchainSubgraphs: argv.offchainSubgraphs.map( diff --git a/packages/indexer-agent/src/index.ts b/packages/indexer-agent/src/index.ts index c6fa21074..675cb0051 100644 --- a/packages/indexer-agent/src/index.ts +++ b/packages/indexer-agent/src/index.ts @@ -7,24 +7,14 @@ import { AgentOptions, run, } from './commands/start' -import { startMultiNetwork } from './commands/start-multi-network' -import { parseNetworkSpecifications } from '@graphprotocol/indexer-common' - -const MULTINETWORK_MODE: boolean = - !!process.env.INDEXER_AGENT_MULTINETWORK_MODE && - process.env.INDEXER_AGENT_MULTINETWORK_MODE.toLowerCase() !== 'false' +import { parseNetworkSpecification } from '@graphprotocol/indexer-common' function parseArguments(): AgentOptions { let builder = yargs.scriptName('indexer-agent').env('INDEXER_AGENT') // Dynamic argument parser construction based on network mode - if (MULTINETWORK_MODE) { - console.log('Starting the Indexer Agent in multi-network mode') - builder = builder.command(startMultiNetwork) - } else { - console.log('Starting the Indexer Agent in single-network mode') - builder = builder.command(start) - } + console.log('Starting the Indexer Agent') + builder = builder.command(start) return ( builder @@ -53,14 +43,15 @@ async function processArgumentsAndRun(args: AgentOptions): Promise { async: false, level: args.logLevel, }) - if (MULTINETWORK_MODE) { - const specifications = parseNetworkSpecifications(args, logger) - await run(args, specifications, logger) + + let specification + if (args.dir || args['network-specifications-directory']) { + specification = parseNetworkSpecification(args, logger) } else { + specification = await createNetworkSpecification(args, logger) reviewArgumentsForWarnings(args, logger) - const specification = await createNetworkSpecification(args, logger) - await run(args, [specification], logger) } + await run(args, specification!, logger) } async function main(): Promise { diff --git a/packages/indexer-agent/src/syncing-server.ts b/packages/indexer-agent/src/syncing-server.ts index 2c08da48b..2920eb7e0 100644 --- a/packages/indexer-agent/src/syncing-server.ts +++ b/packages/indexer-agent/src/syncing-server.ts @@ -5,21 +5,17 @@ import bodyParser from 'body-parser' import morgan from 'morgan' import { Logger } from '@graphprotocol/common-ts' import { parse } from 'graphql' -import { - NetworkMapped, - SubgraphClient, - resolveChainId, -} from '@graphprotocol/indexer-common' +import { SubgraphClient, resolveChainId } from '@graphprotocol/indexer-common' export interface CreateSyncingServerOptions { logger: Logger - networkSubgraphs: NetworkMapped + networkSubgraph: SubgraphClient port: number } export const createSyncingServer = async ({ logger, - networkSubgraphs, + networkSubgraph, port, }: CreateSyncingServerOptions): Promise => { logger = logger.child({ component: 'SyncingServer' }) @@ -64,7 +60,6 @@ export const createSyncingServer = async ({ .send(`Unknown network identifier: '${unvalidatedNetworkIdentifier}'`) } - const networkSubgraph = networkSubgraphs[networkIdentifier] if (!networkSubgraph) { return res .status(404) diff --git a/packages/indexer-agent/src/types.ts b/packages/indexer-agent/src/types.ts index f71f9d295..b945bae59 100644 --- a/packages/indexer-agent/src/types.ts +++ b/packages/indexer-agent/src/types.ts @@ -18,9 +18,9 @@ export interface AgentConfigs { logger: Logger metrics: Metrics graphNode: GraphNode - operators: Operator[] + network: Network + operator: Operator indexerManagement: IndexerManagementClient - networks: Network[] deploymentManagement: DeploymentManagementMode autoMigrationSupport: boolean offchainSubgraphs: SubgraphDeploymentID[] diff --git a/packages/indexer-cli/src/__tests__/cli.test.ts b/packages/indexer-cli/src/__tests__/cli.test.ts index 40fcda30c..9a6178b62 100644 --- a/packages/indexer-cli/src/__tests__/cli.test.ts +++ b/packages/indexer-cli/src/__tests__/cli.test.ts @@ -1,10 +1,10 @@ -import { cliTest, setupMultiNetworks, teardown } from './util' +import { cliTest, setupSingleNetwork, teardown } from './util' import path from 'path' const baseDir = path.join(__dirname) describe('Indexer cli tests', () => { - beforeEach(setupMultiNetworks) + beforeEach(setupSingleNetwork) afterEach(teardown) describe('General', () => { diff --git a/packages/indexer-cli/src/__tests__/indexer/actions.test.ts b/packages/indexer-cli/src/__tests__/indexer/actions.test.ts index 6d432b3d4..00d90538b 100644 --- a/packages/indexer-cli/src/__tests__/indexer/actions.test.ts +++ b/packages/indexer-cli/src/__tests__/indexer/actions.test.ts @@ -2,7 +2,7 @@ import { cliTest, deleteFromAllTables, seedActions, - setupMultiNetworks, + setupSingleNetwork, teardown, } from '../util' import path from 'path' @@ -10,7 +10,7 @@ import path from 'path' const baseDir = path.join(__dirname, '..') describe('Indexer actions tests', () => { describe('With indexer management server', () => { - beforeAll(setupMultiNetworks) + beforeAll(setupSingleNetwork) afterAll(teardown) beforeEach(seedActions) afterEach(deleteFromAllTables) diff --git a/packages/indexer-cli/src/__tests__/indexer/cost.test.ts b/packages/indexer-cli/src/__tests__/indexer/cost.test.ts index 4b08860c7..329560bba 100644 --- a/packages/indexer-cli/src/__tests__/indexer/cost.test.ts +++ b/packages/indexer-cli/src/__tests__/indexer/cost.test.ts @@ -5,7 +5,6 @@ import { deleteFromAllTables, seedCostModels, setupSingleNetwork, - setupMultiNetworks, } from '../util' import path from 'path' @@ -217,7 +216,7 @@ describe('Indexer cost tests singleNetwork', () => { }) describe('Indexer cost tests multiNetworks', () => { - beforeAll(setupMultiNetworks) + beforeAll(setupSingleNetwork) afterAll(teardown) beforeEach(seedCostModels) afterEach(deleteFromAllTables) diff --git a/packages/indexer-cli/src/__tests__/indexer/rules.test.ts b/packages/indexer-cli/src/__tests__/indexer/rules.test.ts index 708440e35..d32231de5 100644 --- a/packages/indexer-cli/src/__tests__/indexer/rules.test.ts +++ b/packages/indexer-cli/src/__tests__/indexer/rules.test.ts @@ -4,7 +4,7 @@ import { teardown, deleteFromAllTables, seedIndexingRules, - setupMultiNetworks, + setupSingleNetwork, } from '../util' import path from 'path' @@ -12,7 +12,7 @@ const baseDir = path.join(__dirname, '..') describe('Indexer rules tests', () => { describe('With indexer management server', () => { - beforeAll(setupMultiNetworks) + beforeAll(setupSingleNetwork) afterAll(teardown) beforeEach(seedIndexingRules) afterEach(deleteFromAllTables) diff --git a/packages/indexer-cli/src/__tests__/util.ts b/packages/indexer-cli/src/__tests__/util.ts index 273a3d649..502bc9689 100644 --- a/packages/indexer-cli/src/__tests__/util.ts +++ b/packages/indexer-cli/src/__tests__/util.ts @@ -19,7 +19,6 @@ import { IndexerManagementModels, IndexingDecisionBasis, loadTestYamlConfig, - MultiNetworks, Network, QueryFeeModels, specification, @@ -56,15 +55,11 @@ let metrics: Metrics const yamlObj = loadTestYamlConfig() const testNetworkSpecification = specification.NetworkSpecification.parse(yamlObj) -export const setupMultiNetworks = async () => { - return await setup(true) -} - export const setupSingleNetwork = async () => { - return await setup(false) + return await setup() } -export const setup = async (multiNetworksEnabled: boolean) => { +export const setup = async () => { logger = createLogger({ name: 'Setup', async: false, @@ -100,13 +95,6 @@ export const setup = async (multiNetworksEnabled: boolean) => { const fakeMainnetNetwork = cloneDeep(network) as Network fakeMainnetNetwork.specification.networkIdentifier = 'eip155:1' - const multiNetworks = multiNetworksEnabled - ? new MultiNetworks( - [network, fakeMainnetNetwork], - (n: Network) => n.specification.networkIdentifier, - ) - : new MultiNetworks([network], (n: Network) => n.specification.networkIdentifier) - const defaults: IndexerManagementDefaults = { globalIndexingRule: { allocationAmount: parseGRT('100'), @@ -121,7 +109,7 @@ export const setup = async (multiNetworksEnabled: boolean) => { graphNode, logger, defaults, - multiNetworks, + network, }) server = await createIndexerManagementServer({ diff --git a/packages/indexer-common/src/index.ts b/packages/indexer-common/src/index.ts index ab3eedd97..e22722403 100644 --- a/packages/indexer-common/src/index.ts +++ b/packages/indexer-common/src/index.ts @@ -15,5 +15,4 @@ export * from './types' export * from './utils' export * from './parsers' export * as specification from './network-specification' -export * from './multi-networks' export * from './sequential-timer' diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts index c97df0722..ba9b3848f 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts @@ -14,7 +14,6 @@ import { IndexingRuleAttributes, } from '../models' import { defineQueryFeeModels, specification as spec } from '../../index' -import { networkIsL1, networkIsL2 } from '../types' import { fetchIndexingRules, upsertIndexingRule } from '../rules' import { SubgraphFreshnessChecker, SubgraphIdentifierType } from '../../subgraphs' import { ActionManager } from '../actions' @@ -479,51 +478,3 @@ describe.skip('Monitor: CI', () => { await expect(deployments.length).toBeGreaterThan(500) }, 40000) }) - -describe('Network layer detection', () => { - interface NetworkLayer { - name: string - l1: boolean - l2: boolean - } - - // Should be true for L1 and false for L2 - const l1Networks: NetworkLayer[] = ['mainnet', 'eip155:1', 'sepolia'].map( - (name: string) => ({ name, l1: true, l2: false }), - ) - - // Should be false for L1 and true for L2 - const l2Networks: NetworkLayer[] = [ - 'arbitrum-one', - 'eip155:42161', - 'eip155:421614', - ].map((name: string) => ({ name, l1: false, l2: true })) - - // Those will be false for L1 and L2 - const nonProtocolNetworks: NetworkLayer[] = [ - 'fantom', - 'eip155:250', - 'hardhat', - 'eip155:1337', - 'matic', - 'eip155:137', - 'gnosis', - 'eip155:100', - ].map((name: string) => ({ name, l1: false, l2: false })) - - const testCases = [...l1Networks, ...l2Networks, ...nonProtocolNetworks] - - test.each(testCases)('Can detect network layer [$name]', (network) => { - expect(networkIsL1(network.name)).toStrictEqual(network.l1) - expect(networkIsL2(network.name)).toStrictEqual(network.l2) - }) - - const invalidTProtocolNetworkNames = ['invalid-name', 'eip155:9999'] - - test.each(invalidTProtocolNetworkNames)( - 'Throws error when protocol network is unknown [%s]', - (invalidProtocolNetworkName) => { - expect(() => networkIsL1(invalidProtocolNetworkName)).toThrow() - }, - ) -}) diff --git a/packages/indexer-common/src/indexer-management/__tests__/util.ts b/packages/indexer-common/src/indexer-management/__tests__/util.ts index a4a14c605..1a9fe6f4a 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/util.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/util.ts @@ -9,7 +9,6 @@ import { IndexerManagementClient, IndexerManagementDefaults, loadTestYamlConfig, - MultiNetworks, Network, specification, } from '@graphprotocol/indexer-common' @@ -66,17 +65,12 @@ export const createTestManagementClient = async ( network.specification.networkIdentifier = networkIdentifierOverride } - const multiNetworks = new MultiNetworks( - [network], - (n: Network) => n.specification.networkIdentifier, - ) - return await createIndexerManagementClient({ models: managementModels, graphNode, logger, defaults, - multiNetworks, + network, }) } diff --git a/packages/indexer-common/src/indexer-management/actions.ts b/packages/indexer-common/src/indexer-management/actions.ts index 237dd718f..9eebab11a 100644 --- a/packages/indexer-common/src/indexer-management/actions.ts +++ b/packages/indexer-common/src/indexer-management/actions.ts @@ -13,8 +13,6 @@ import { IndexerErrorCode, IndexerManagementModels, isActionFailure, - MultiNetworks, - NetworkMapped, Network, OrderDirection, GraphNode, @@ -23,37 +21,34 @@ import { import { Order, Transaction } from 'sequelize' import { Eventual, join, Logger } from '@graphprotocol/common-ts' -import groupBy from 'lodash.groupby' export class ActionManager { - declare multiNetworks: MultiNetworks + declare network: Network declare logger: Logger declare models: IndexerManagementModels - declare allocationManagers: NetworkMapped + declare allocationManager: AllocationManager executeBatchActionsPromise: Promise | undefined static async create( - multiNetworks: MultiNetworks, + network: Network, logger: Logger, models: IndexerManagementModels, graphNode: GraphNode, ): Promise { const actionManager = new ActionManager() - actionManager.multiNetworks = multiNetworks + actionManager.network = network actionManager.logger = logger.child({ component: 'ActionManager' }) actionManager.models = models - actionManager.allocationManagers = await multiNetworks.map(async (network) => { - return new AllocationManager( - logger.child({ - component: 'AllocationManager', - protocolNetwork: network.specification.networkIdentifier, - }), - models, - graphNode, - network, - ) - }) + actionManager.allocationManager = new AllocationManager( + logger.child({ + component: 'AllocationManager', + protocolNetwork: network.specification.networkIdentifier, + }), + models, + graphNode, + network, + ) logger.info('Begin monitoring the queue for approved actions to execute') await actionManager.monitorQueue() @@ -147,62 +142,52 @@ export class ActionManager { join({ approvedActions }).pipe(async ({ approvedActions }) => { logger.debug('Approved actions found, evaluating batch') - const approvedActionsByNetwork: NetworkMapped = groupBy( - approvedActions, - (action: Action) => action.protocolNetwork, - ) - await this.multiNetworks.mapNetworkMapped( - approvedActionsByNetwork, - async (network: Network, approvedActions: Action[]) => { - const networkLogger = logger.child({ - protocolNetwork: network.specification.networkIdentifier, - indexer: network.specification.indexerOptions.address, - operator: network.transactionManager.wallet.address, + const networkLogger = logger.child({ + protocolNetwork: this.network.specification.networkIdentifier, + indexer: this.network.specification.indexerOptions.address, + }) + + if (await this.batchReady(approvedActions, this.network, networkLogger)) { + const paused = await this.network.paused.value() + const isOperator = await this.network.isOperator.value() + networkLogger.debug('Batch ready, preparing to execute', { + paused, + isOperator, + protocolNetwork: this.network.specification.networkIdentifier, + }) + // Do nothing else if the network is paused + if (paused) { + networkLogger.info( + `The network is currently paused, not doing anything until it resumes`, + ) + return + } + + // Do nothing if we're not authorized as an operator for the indexer + if (!isOperator) { + networkLogger.error(`Not authorized as an operator for the indexer`, { + err: indexerError(IndexerErrorCode.IE034), }) + return + } - if (await this.batchReady(approvedActions, network, networkLogger)) { - const paused = await network.paused.value() - const isOperator = await network.isOperator.value() - networkLogger.debug('Batch ready, preparing to execute', { - paused, - isOperator, - protocolNetwork: network.specification.networkIdentifier, - }) - // Do nothing else if the network is paused - if (paused) { - networkLogger.info( - `The network is currently paused, not doing anything until it resumes`, - ) - return - } - - // Do nothing if we're not authorized as an operator for the indexer - if (!isOperator) { - networkLogger.error(`Not authorized as an operator for the indexer`, { - err: indexerError(IndexerErrorCode.IE034), - }) - return - } - - networkLogger.info('Executing batch of approved actions', { - actions: approvedActions, - note: 'If actions were approved very recently they may be missing from this batch', - }) + networkLogger.info('Executing batch of approved actions', { + actions: approvedActions, + note: 'If actions were approved very recently they may be missing from this batch', + }) - try { - const attemptedActions = await this.executeApprovedActions(network) - networkLogger.trace('Attempted to execute all approved actions', { - actions: attemptedActions, - }) - } catch (error) { - networkLogger.error('Failed to execute batch of approved actions', { - error, - }) - } - } - }, - ) + try { + const attemptedActions = await this.executeApprovedActions(this.network) + networkLogger.trace('Attempted to execute all approved actions', { + actions: attemptedActions, + }) + } catch (error) { + networkLogger.error('Failed to execute batch of approved actions', { + error, + }) + } + } }) } @@ -352,13 +337,10 @@ export class ActionManager { startTimeMs: Date.now() - batchStartTime, }) - const allocationManager = - this.allocationManagers[network.specification.networkIdentifier] - let results try { // This will return all results if successful, if failed it will return the failed actions - results = await allocationManager.executeBatch(prioritizedActions) + results = await this.allocationManager.executeBatch(prioritizedActions) logger.debug('Completed batch action execution', { results, endTimeMs: Date.now() - batchStartTime, diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index 280eb62cb..2afec1d57 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -14,7 +14,7 @@ import poiDisputeResolvers from './resolvers/poi-disputes' import statusResolvers from './resolvers/indexer-status' import { BigNumber } from 'ethers' import { GraphNode } from '../graph-node' -import { ActionManager, MultiNetworks, Network } from '@graphprotocol/indexer-common' +import { ActionManager, Network } from '@graphprotocol/indexer-common' export interface IndexerManagementResolverContext { models: IndexerManagementModels @@ -22,7 +22,7 @@ export interface IndexerManagementResolverContext { logger: Logger defaults: IndexerManagementDefaults actionManager: ActionManager | undefined - multiNetworks: MultiNetworks | undefined + network: Network | undefined } const SCHEMA_SDL = gql` @@ -452,7 +452,7 @@ export interface IndexerManagementClientOptions { logger: Logger models: IndexerManagementModels graphNode: GraphNode - multiNetworks: MultiNetworks | undefined + network: Network | undefined defaults: IndexerManagementDefaults } @@ -468,12 +468,10 @@ export class IndexerManagementClient extends Client { } } -// TODO:L2: Put the IndexerManagementClient creation inside the Agent, and receive -// MultiNetworks from it export const createIndexerManagementClient = async ( options: IndexerManagementClientOptions, ): Promise => { - const { models, graphNode, logger, defaults, multiNetworks } = options + const { models, graphNode, logger, defaults, network } = options const schema = buildSchema(print(SCHEMA_SDL)) const resolvers = { ...indexingRuleResolvers, @@ -484,8 +482,8 @@ export const createIndexerManagementClient = async ( ...actionResolvers, } - const actionManager = multiNetworks - ? await ActionManager.create(multiNetworks, logger, models, graphNode) + const actionManager = network + ? await ActionManager.create(network, logger, models, graphNode) : undefined const context: IndexerManagementResolverContext = { @@ -493,7 +491,7 @@ export const createIndexerManagementClient = async ( graphNode, defaults, logger: logger.child({ component: 'IndexerManagementClient' }), - multiNetworks, + network, actionManager, } diff --git a/packages/indexer-common/src/indexer-management/monitor.ts b/packages/indexer-common/src/indexer-management/monitor.ts index e09e43fef..69219258c 100644 --- a/packages/indexer-common/src/indexer-management/monitor.ts +++ b/packages/indexer-common/src/indexer-management/monitor.ts @@ -16,7 +16,6 @@ import { BlockPointer, resolveChainId, resolveChainAlias, - TransferredSubgraphDeployment, sequentialTimerReduce, } from '@graphprotocol/indexer-common' import { @@ -561,93 +560,6 @@ export class NetworkMonitor { } } - async transferredDeployments(): Promise { - this.logger.debug('Querying the Network for transferred subgraph deployments') - try { - const result = await this.networkSubgraph.checkedQuery( - // TODO: Consider querying for the same time range as the Agent's evaluation, limiting - // results to recent transfers. - gql` - { - subgraphs( - where: { startedTransferToL2: true } - orderBy: startedTransferToL2At - orderDirection: asc - ) { - id - idOnL1 - idOnL2 - startedTransferToL2 - startedTransferToL2At - startedTransferToL2AtBlockNumber - startedTransferToL2AtTx - transferredToL2 - transferredToL2At - transferredToL2AtBlockNumber - transferredToL2AtTx - versions { - subgraphDeployment { - ipfsHash - } - } - } - } - `, - ) - - if (result.error) { - throw result.error - } - - const transferredDeployments = result.data.subgraphs - - // There may be no transferred subgraphs, handle gracefully - if (transferredDeployments.length == 0) { - this.logger.warn( - 'Failed to query subgraph deployments transferred to L2: no deployments found', - ) - throw new Error('No transferred subgraph deployments returned') - } - - // Flatten multiple subgraphDeployment versions into a single `TransferredSubgraphDeployment` object - // TODO: We could use `zod` to parse GraphQL responses into the expected type - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return transferredDeployments.flatMap((deployment: any) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return deployment.versions.map((version: any) => { - return { - id: deployment.id, - idOnL1: deployment.idOnL1, - idOnL2: deployment.idOnL2, - startedTransferToL2: deployment.startedTransferToL2, - startedTransferToL2At: BigNumber.from(deployment.startedTransferToL2At), - startedTransferToL2AtBlockNumber: BigNumber.from( - deployment.startedTransferToL2AtBlockNumber, - ), - startedTransferToL2AtTx: deployment.startedTransferToL2AtTx, - transferredToL2: deployment.transferredToL2, - transferredToL2At: deployment.transferredToL2At - ? BigNumber.from(deployment.transferredToL2At) - : null, - transferredToL2AtTx: deployment.transferredToL2AtTx, - transferredToL2AtBlockNumber: deployment.transferredToL2AtBlockNumber - ? BigNumber.from(deployment.transferredToL2AtBlockNumber) - : null, - ipfsHash: version.subgraphDeployment.ipfsHash, - protocolNetwork: this.networkCAIPID, - ready: null, - } - }) - }) - } catch (err) { - const error = indexerError(IndexerErrorCode.IE009, err.message) - this.logger.error(`Failed to query transferred subgraph deployments`, { - error, - }) - throw error - } - } - async subgraphDeployments(): Promise { const deployments: SubgraphDeployment[] = [] const queryProgress = { diff --git a/packages/indexer-common/src/indexer-management/resolvers/actions.ts b/packages/indexer-common/src/indexer-management/resolvers/actions.ts index 1e246754c..6890797fd 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/actions.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/actions.ts @@ -11,15 +11,12 @@ import { ActionType, ActionUpdateInput, IndexerManagementModels, - Network, - NetworkMapped, OrderDirection, validateActionInputs, validateNetworkIdentifier, } from '@graphprotocol/indexer-common' import { literal, Op, Transaction } from 'sequelize' import { ActionManager } from '../actions' -import groupBy from 'lodash.groupby' // Perform insert, update, or no-op depending on existing queue data // INSERT - No item in the queue yet targeting this deploymentID @@ -151,13 +148,13 @@ export default { queueActions: async ( { actions }: { actions: ActionInput[] }, - { actionManager, logger, multiNetworks, models }: IndexerManagementResolverContext, + { actionManager, logger, network, models }: IndexerManagementResolverContext, ): Promise => { logger.debug(`Execute 'queueActions' mutation`, { actions, }) - if (!actionManager || !multiNetworks) { + if (!actionManager || !network) { throw Error('IndexerManagementClient must be in `network` mode to modify actions') } @@ -171,11 +168,7 @@ export default { }) // Let Network Monitors validate actions based on their protocol networks - await multiNetworks.mapNetworkMapped( - groupBy(actions, (action) => action.protocolNetwork), - (network: Network, actions: ActionInput[]) => - validateActionInputs(actions, network.networkMonitor, logger), - ) + await validateActionInputs(actions, network.networkMonitor, logger) let results: ActionResult[] = [] @@ -360,23 +353,20 @@ export default { if (!actionManager) { throw Error('IndexerManagementClient must be in `network` mode to modify actions') } - const result: NetworkMapped = await actionManager.multiNetworks.map( - async (network: Network) => { - logger.debug(`Execute 'executeApprovedActions' mutation`, { - protocolNetwork: network.specification.networkIdentifier, - }) - try { - return await actionManager.executeApprovedActions(network) - } catch (error) { - logger.error('Failed to execute approved actions for network', { - protocolNetwork: network.specification.networkIdentifier, - error, - }) - return [] - } - }, - ) - return Object.values(result).flat() + const { network } = actionManager + logger.debug(`Execute 'executeApprovedActions' mutation`, { + protocolNetwork: network.specification.networkIdentifier, + }) + try { + const result = await actionManager.executeApprovedActions(network) + return result + } catch (error) { + logger.error('Failed to execute approved actions for network', { + protocolNetwork: network.specification.networkIdentifier, + error, + }) + return [] + } }, } diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index 5d2e72690..b412beffb 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -1,4 +1,4 @@ -import { epochElapsedBlocks, Network } from '@graphprotocol/indexer-common' +import { epochElapsedBlocks } from '@graphprotocol/indexer-common' /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ /* eslint-disable @typescript-eslint/ban-types */ @@ -29,7 +29,6 @@ import { SubgraphIdentifierType, uniqueAllocationID, } from '@graphprotocol/indexer-common' -import { extractNetwork } from './utils' interface AllocationFilter { status: 'active' | 'closed' @@ -298,67 +297,68 @@ async function queryAllocations( export default { allocations: async ( { filter }: { filter: AllocationFilter }, - { multiNetworks, logger }: IndexerManagementResolverContext, + { network, logger }: IndexerManagementResolverContext, ): Promise => { logger.debug('Execute allocations() query', { filter, }) - if (!multiNetworks) { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch allocations', ) } - const allocationsByNetwork = await multiNetworks.map( - async (network: Network): Promise => { - // Return early if a different protocol network is specifically requested - if ( - filter.protocolNetwork && - filter.protocolNetwork !== network.specification.networkIdentifier - ) { - return [] - } + // Return early if a different protocol network is specifically requested + if ( + filter.protocolNetwork && + filter.protocolNetwork !== network.specification.networkIdentifier + ) { + return [] + } - const { - networkMonitor, - networkSubgraph, - contracts, - specification: { - indexerOptions: { address }, - }, - } = network - - const [currentEpoch, maxAllocationEpochs, epochLength] = await Promise.all([ - networkMonitor.networkCurrentEpoch(), - contracts.staking.maxAllocationEpochs(), - contracts.epochManager.epochLength(), - ]) - - const allocation = filter.allocation - ? filter.allocation === 'all' - ? null - : toAddress(filter.allocation) - : null - - const variables = { - indexer: toAddress(address), - allocation, - status: filter.status, - } + const { + networkMonitor, + networkSubgraph, + contracts, + specification: { + indexerOptions: { address }, + }, + } = network + + const [currentEpoch, maxAllocationEpochs, epochLength] = await Promise.all([ + networkMonitor.networkCurrentEpoch(), + contracts.staking.maxAllocationEpochs(), + contracts.epochManager.epochLength(), + ]) + + const allocation = filter.allocation + ? filter.allocation === 'all' + ? null + : toAddress(filter.allocation) + : null + + const variables = { + indexer: toAddress(address), + allocation, + status: filter.status, + } - const context = { - currentEpoch: currentEpoch.epochNumber, - currentEpochStartBlock: currentEpoch.startBlockNumber, - currentEpochElapsedBlocks: epochElapsedBlocks(currentEpoch), - latestBlock: currentEpoch.latestBlock, - maxAllocationEpochs, - blocksPerEpoch: epochLength.toNumber(), - avgBlockTime: 13000, - protocolNetwork: network.specification.networkIdentifier, - } + const context = { + currentEpoch: currentEpoch.epochNumber, + currentEpochStartBlock: currentEpoch.startBlockNumber, + currentEpochElapsedBlocks: epochElapsedBlocks(currentEpoch), + latestBlock: currentEpoch.latestBlock, + maxAllocationEpochs, + blocksPerEpoch: epochLength.toNumber(), + avgBlockTime: 13000, + protocolNetwork: network.specification.networkIdentifier, + } - return queryAllocations(logger, networkSubgraph, variables, context) - }, + const allocationsByNetwork = await queryAllocations( + logger, + networkSubgraph, + variables, + context, ) return Object.values(allocationsByNetwork).flat() @@ -374,19 +374,18 @@ export default { amount: string protocolNetwork: string }, - { multiNetworks, graphNode, logger, models }: IndexerManagementResolverContext, + { network, graphNode, logger, models }: IndexerManagementResolverContext, ): Promise => { logger.debug('Execute createAllocation() mutation', { deployment, amount, protocolNetwork, }) - if (!multiNetworks) { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch allocations', ) } - const network = extractNetwork(protocolNetwork, multiNetworks) const networkMonitor = network.networkMonitor const contracts = network.contracts const transactionManager = network.transactionManager @@ -604,25 +603,22 @@ export default { allocation, poi, force, - protocolNetwork, }: { allocation: string poi: string | undefined force: boolean - protocolNetwork: string }, - { logger, models, multiNetworks }: IndexerManagementResolverContext, + { logger, models, network }: IndexerManagementResolverContext, ): Promise => { logger.debug('Execute closeAllocation() mutation', { allocationID: allocation, poi: poi || 'none provided', }) - if (!multiNetworks) { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch allocations', ) } - const network = extractNetwork(protocolNetwork, multiNetworks) const networkMonitor = network.networkMonitor const contracts = network.contracts const transactionManager = network.transactionManager @@ -769,7 +765,7 @@ export default { force: boolean protocolNetwork: string }, - { logger, models, multiNetworks }: IndexerManagementResolverContext, + { logger, models, network }: IndexerManagementResolverContext, ): Promise => { logger = logger.child({ component: 'reallocateAllocationResolver', @@ -782,14 +778,13 @@ export default { force, }) - if (!multiNetworks) { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch allocations', ) } // Obtain the Network object and its associated components and data - const network = extractNetwork(protocolNetwork, multiNetworks) const networkMonitor = network.networkMonitor const contracts = network.contracts const transactionManager = network.transactionManager @@ -1072,18 +1067,17 @@ export default { allocation: string protocolNetwork: string }, - { logger, multiNetworks }: IndexerManagementResolverContext, + { logger, network }: IndexerManagementResolverContext, ): Promise => { logger.debug('Execute collectAllocationReceipts() mutation', { allocationID: allocation, protocolNetwork, }) - if (!multiNetworks) { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to collect receipts for an allocation', ) } - const network = extractNetwork(protocolNetwork, multiNetworks) const networkMonitor = network.networkMonitor const receiptCollector = network.receiptCollector diff --git a/packages/indexer-common/src/indexer-management/resolvers/cost-models.ts b/packages/indexer-common/src/indexer-management/resolvers/cost-models.ts index 45587363e..54e015714 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/cost-models.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/cost-models.ts @@ -80,14 +80,11 @@ export default { setCostModel: async ( { costModel }: { deployment: string; costModel: GraphQLCostModel }, - { models, multiNetworks }: IndexerManagementResolverContext, + { models, network }: IndexerManagementResolverContext, ): Promise => { - if (!multiNetworks) { + if (!network) { throw new Error('No network configuration available') } - if (Object.keys(multiNetworks.inner).length !== 1) { - throw Error('Must be in single network mode to set cost models') - } const update = parseGraphQLCostModel(costModel) // Validate cost model diff --git a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts index c040113a2..67784f502 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts @@ -10,7 +10,6 @@ import { Network, validateNetworkIdentifier, } from '@graphprotocol/indexer-common' -import { extractNetwork } from './utils' interface Test { test: (url: string) => string run: (url: string) => Promise @@ -61,17 +60,15 @@ const URL_VALIDATION_TEST: Test = { } export default { - indexerRegistration: async ( - { protocolNetwork: unvalidatedProtocolNetwork }: { protocolNetwork: string }, - { multiNetworks }: IndexerManagementResolverContext, - ): Promise => { - if (!multiNetworks) { + indexerRegistration: async ({ + network, + }: IndexerManagementResolverContext): Promise => { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch indexer registration information', ) } - const network = extractNetwork(unvalidatedProtocolNetwork, multiNetworks) const protocolNetwork = network.specification.networkIdentifier const address = network.specification.indexerOptions.address const contracts = network.contracts @@ -110,17 +107,16 @@ export default { })) }, - indexerAllocations: async ( - { protocolNetwork }: { protocolNetwork: string }, - { multiNetworks, logger }: IndexerManagementResolverContext, - ): Promise => { - if (!multiNetworks) { + indexerAllocations: async ({ + network, + logger, + }: IndexerManagementResolverContext): Promise => { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch indexer allocations', ) } - const network = extractNetwork(protocolNetwork, multiNetworks) const address = network.specification.indexerOptions.address try { @@ -188,9 +184,9 @@ export default { indexerEndpoints: async ( { protocolNetwork: unvalidatedProtocolNetwork }: { protocolNetwork: string | null }, - { multiNetworks, logger }: IndexerManagementResolverContext, + { network, logger }: IndexerManagementResolverContext, ): Promise => { - if (!multiNetworks) { + if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch indexer endpoints', ) @@ -209,25 +205,22 @@ export default { ) } - await multiNetworks.map(async (network: Network) => { - // Skip if this query asks for another protocol network - if ( - networkIdentifier && - networkIdentifier !== network.specification.networkIdentifier - ) { - return - } - try { - const networkEndpoints = await endpointForNetwork(network) - endpoints.push(networkEndpoints) - } catch (err) { - // Ignore endpoints for this network - logger?.warn(`Failed to detect service endpoints for network`, { - err, - protocolNetwork: network.specification.networkIdentifier, - }) - } - }) + if ( + networkIdentifier && + networkIdentifier !== network.specification.networkIdentifier + ) { + return endpoints + } + try { + const networkEndpoints = await endpointForNetwork(network) + endpoints.push(networkEndpoints) + } catch (err) { + // Ignore endpoints for this network + logger?.warn(`Failed to detect service endpoints for network`, { + err, + protocolNetwork: network.specification.networkIdentifier, + }) + } return endpoints }, } diff --git a/packages/indexer-common/src/indexer-management/resolvers/utils.ts b/packages/indexer-common/src/indexer-management/resolvers/utils.ts deleted file mode 100644 index baa061fdf..000000000 --- a/packages/indexer-common/src/indexer-management/resolvers/utils.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { - MultiNetworks, - Network, - validateNetworkIdentifier, -} from '@graphprotocol/indexer-common' - -export function extractNetwork( - unvalidatedNetworkIdentifier: string, - multiNetworks: MultiNetworks, -): Network { - let networkIdentifier: string - try { - networkIdentifier = validateNetworkIdentifier(unvalidatedNetworkIdentifier) - } catch (parseError) { - throw new Error( - `Invalid protocol network identifier: '${unvalidatedNetworkIdentifier}'. Error: ${parseError}`, - ) - } - const network = multiNetworks.inner[networkIdentifier] - if (!network) { - throw new Error( - `Could not find a configured protocol network named ${networkIdentifier}`, - ) - } - return network -} diff --git a/packages/indexer-common/src/indexer-management/types.ts b/packages/indexer-common/src/indexer-management/types.ts index beeab7f6a..cf801f4cb 100644 --- a/packages/indexer-common/src/indexer-management/types.ts +++ b/packages/indexer-common/src/indexer-management/types.ts @@ -290,17 +290,3 @@ export async function validateProviderNetworkIdentifier( throw new Error(errorMsg) } } - -// Convenience function to check if a given network identifier is a supported Layer-1 protocol network -export function networkIsL1(networkIdentifier: string): boolean { - // Normalize network identifier - networkIdentifier = resolveChainId(networkIdentifier) - return networkIdentifier === 'eip155:1' || networkIdentifier === 'eip155:11155111' -} - -// Convenience function to check if a given network identifier is a supported Layer-2 protocol network -export function networkIsL2(networkIdentifier: string): boolean { - // Normalize network identifier - networkIdentifier = resolveChainId(networkIdentifier) - return networkIdentifier === 'eip155:42161' || networkIdentifier === 'eip155:421614' -} diff --git a/packages/indexer-common/src/multi-networks.ts b/packages/indexer-common/src/multi-networks.ts deleted file mode 100644 index f305d2087..000000000 --- a/packages/indexer-common/src/multi-networks.ts +++ /dev/null @@ -1,111 +0,0 @@ -import pReduce from 'p-reduce' -import isEqual from 'lodash.isequal' -import xor from 'lodash.xor' - -// A mapping of different values of type T keyed by their network identifiers -export type NetworkMapped = Record - -// Function to extract the network identifier from a value of type T -type NetworkIdentity = (element: T) => string - -// Wrapper type for performing calls over multiple values of any type, most notably -// Network and Operator instances. -// All public-facing methods should return a `NetworkMapped` or `void`. -export class MultiNetworks { - inner: NetworkMapped - constructor(elements: T[], networkIdentity: NetworkIdentity) { - if (elements.length === 0) { - throw new Error('MultiNetworks component was initialized with empty values') - } - - function reducer(accumulator: NetworkMapped, current: T): NetworkMapped { - const key = networkIdentity(current) - if (key in accumulator) { - throw new Error( - `Duplicate network identifier found while mapping value's network: ${key}`, - ) - } - // TODO: parse and validate network identifiers to standardize them - accumulator[key] = current - return accumulator - } - this.inner = elements.reduce(reducer, {}) - } - - private checkEqualKeys(a: NetworkMapped, b: NetworkMapped) { - const aKeys = Object.keys(a) - const bKeys = Object.keys(b) - if (!isEqual(aKeys, bKeys)) { - const differentKeys = xor(aKeys, bKeys) - throw new Error(`Network Mapped objects have different keys: ${differentKeys}`) - } - } - - async map(func: (value: T) => Promise): Promise> { - const entries: [string, T][] = Object.entries(this.inner) - return pReduce( - entries, - async (acc, pair) => { - const [networkIdentifier, element]: [string, T] = pair - const result = await func(element) - acc[networkIdentifier] = result - return acc - }, - {} as NetworkMapped, - ) - } - - zip(a: NetworkMapped, b: NetworkMapped): NetworkMapped<[U, V]> { - this.checkEqualKeys(a, b) - const result = {} as NetworkMapped<[U, V]> - for (const key in a) { - result[key] = [a[key], b[key]] - } - return result - } - - zip3( - a: NetworkMapped, - b: NetworkMapped, - c: NetworkMapped, - ): NetworkMapped<[U, V, W]> { - this.checkEqualKeys(a, b) - const result = {} as NetworkMapped<[U, V, W]> - for (const key in a) { - result[key] = [a[key], b[key], c[key]] - } - return result - } - - zip4( - a: NetworkMapped, - b: NetworkMapped, - c: NetworkMapped, - d: NetworkMapped, - ): NetworkMapped<[U, V, W, Y]> { - this.checkEqualKeys(a, b) - const result = {} as NetworkMapped<[U, V, W, Y]> - for (const key in a) { - result[key] = [a[key], b[key], c[key], d[key]] - } - return result - } - - async mapNetworkMapped( - nmap: NetworkMapped, - func: (inner: T, value: U) => Promise, - ): Promise> { - return pReduce( - Object.entries(nmap), - async (acc, [networkIdentifier, value]: [string, U]) => { - const inner = this.inner[networkIdentifier] - if (!inner) { - throw new Error(`Network identifier not found: ${networkIdentifier}`) - } - acc[networkIdentifier] = await func(inner, value) - return acc - }, - {} as NetworkMapped, - ) - } -} diff --git a/packages/indexer-common/src/parsers/test-utils.ts b/packages/indexer-common/src/parsers/test-utils.ts index 3463cbdec..0c80fc416 100644 --- a/packages/indexer-common/src/parsers/test-utils.ts +++ b/packages/indexer-common/src/parsers/test-utils.ts @@ -72,18 +72,22 @@ function parseYamlFile(filePath: string): NetworkSpecification { } } -function parseYamlFiles(filePaths: string[]): NetworkSpecification[] { - return filePaths.map(parseYamlFile) -} - -export function parseNetworkSpecifications( +export function parseNetworkSpecification( // eslint-disable-next-line @typescript-eslint/no-explicit-any argv: any, logger: Logger, -): NetworkSpecification[] { +): NetworkSpecification | undefined { const dir: string = argv.dir || argv['network-specifications-directory'] const yamlFiles = scanDirectoryForYamlFiles(dir, logger) - return parseYamlFiles(yamlFiles) + if (yamlFiles.length === 0) { + logger.info('No network specification files found in the provided directory') + return undefined + } else if (yamlFiles.length === 1) { + logger.info(`Found yaml config at ${dir}/${yamlFiles[0]} (ignoring others})`) + return parseYamlFile(yamlFiles[0]) + } else { + throw new Error(`Multiple network specification files found in ${dir}.`) + } } function scanDirectoryForYamlFiles(directoryPath: string, logger: Logger): string[] { @@ -91,13 +95,13 @@ function scanDirectoryForYamlFiles(directoryPath: string, logger: Logger): strin // Check if the directory exists if (!fs.existsSync(directoryPath)) { - throw new Error(`Directory does not exist: ${directoryPath}`) + throw new Error(`Directory does not exist: ${directoryPath} `) } // Check if the provided path is a directory const isDirectory = fs.lstatSync(directoryPath).isDirectory() if (!isDirectory) { - throw new Error(`Provided path is not a directory: ${directoryPath}`) + throw new Error(`Provided path is not a directory: ${directoryPath} `) } // Read the directory @@ -124,7 +128,7 @@ function scanDirectoryForYamlFiles(directoryPath: string, logger: Logger): strin fs.accessSync(filePath, fs.constants.R_OK) yamlFiles.push(filePath) } catch (error) { - throw new Error(`Cannot read file: ${filePath}`) + throw new Error(`Cannot read file: ${filePath} `) } } } @@ -132,7 +136,7 @@ function scanDirectoryForYamlFiles(directoryPath: string, logger: Logger): strin // Check if at least one YAMl file was found if (yamlFiles.length === 0) { throw new Error( - `No YAML file was found in '${directoryPath}'. At least one file is required.`, + `No YAML file was found in '${directoryPath}'.At least one file is required.`, ) } diff --git a/packages/indexer-common/src/types.ts b/packages/indexer-common/src/types.ts index cfd62747a..cb6f81fd4 100644 --- a/packages/indexer-common/src/types.ts +++ b/packages/indexer-common/src/types.ts @@ -66,25 +66,6 @@ export interface SubgraphDeployment { protocolNetwork: string } -// L1 Network Subgraph will always return `null` for the -// `transferredToL2*` set of fields -export interface TransferredSubgraphDeployment { - id: string - idOnL1: string - idOnL2: string - startedTransferToL2L: boolean - startedTransferToL2At: BigNumber - startedTransferToL2AtBlockNumber: BigNumber - startedTransferToL2AtTx: string - transferredToL2: boolean | null - transferredToL2At: BigNumber | null - transferredToL2AtTx: string | null - transferredToL2AtBlockNumber: BigNumber | null - ipfsHash: string - protocolNetwork: string - ready: boolean | null -} - export enum TransactionType { ZERO, TWO, From d2b41c64c29d3c3c1c7e36f25ed29ba616b8ee50 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Mon, 10 Feb 2025 11:51:48 -0800 Subject: [PATCH 02/11] agent,common: fix status endpoint --- packages/indexer-agent/src/commands/start.ts | 2 +- .../src/indexer-management/client.ts | 4 ++-- .../resolvers/indexer-status.ts | 17 +++++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 004e48077..a4453f3de 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -548,7 +548,7 @@ export async function run( queryInterface: sequelize.getQueryInterface(), logger, graphNodeAdminEndpoint: argv.graphNodeAdminEndpoint, - networkSpecifications: networkSpecification, + networkSpecifications: [networkSpecification], graphNode: graphNode, }, storage: new SequelizeStorage({ sequelize }), diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index 2afec1d57..6907179cf 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -22,7 +22,7 @@ export interface IndexerManagementResolverContext { logger: Logger defaults: IndexerManagementDefaults actionManager: ActionManager | undefined - network: Network | undefined + network: Network } const SCHEMA_SDL = gql` @@ -452,7 +452,7 @@ export interface IndexerManagementClientOptions { logger: Logger models: IndexerManagementModels graphNode: GraphNode - network: Network | undefined + network: Network defaults: IndexerManagementDefaults } diff --git a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts index 67784f502..e1e3eae95 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts @@ -60,9 +60,10 @@ const URL_VALIDATION_TEST: Test = { } export default { - indexerRegistration: async ({ - network, - }: IndexerManagementResolverContext): Promise => { + indexerRegistration: async ( + _: { protocolNetwork: string | null }, + { network }: IndexerManagementResolverContext, + ): Promise => { if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch indexer registration information', @@ -97,7 +98,7 @@ export default { }, indexerDeployments: async ( - _: {}, + _: { protocolNetwork: string | null }, { graphNode }: IndexerManagementResolverContext, ): Promise => { const result = await graphNode.indexingStatus([]) @@ -107,10 +108,10 @@ export default { })) }, - indexerAllocations: async ({ - network, - logger, - }: IndexerManagementResolverContext): Promise => { + indexerAllocations: async ( + _: { protocolNetwork: string | null }, + { network, logger }: IndexerManagementResolverContext, + ): Promise => { if (!network) { throw Error( 'IndexerManagementClient must be in `network` mode to fetch indexer allocations', From afe2544dd65010334d94cab6b5f906c6a4ca8e64 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Tue, 11 Feb 2025 08:11:56 -0800 Subject: [PATCH 03/11] cli: remove multinetwork test --- packages/indexer-agent/src/agent.ts | 4 +-- .../src/__tests__/indexer/cost.test.ts | 27 ------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 790143429..373c68c10 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -53,7 +53,7 @@ const deploymentRuleInList = ( rule => rule.identifierType == SubgraphIdentifierType.DEPLOYMENT && new SubgraphDeploymentID(rule.identifier).toString() == - deployment.toString(), + deployment.toString(), ) !== undefined const uniqueDeploymentsOnly = ( @@ -626,7 +626,7 @@ export class Agent { let status = rewardsPool!.referencePOI == allocation.poi || - rewardsPool!.referencePreviousPOI == allocation.poi + rewardsPool!.referencePreviousPOI == allocation.poi ? 'valid' : 'potential' diff --git a/packages/indexer-cli/src/__tests__/indexer/cost.test.ts b/packages/indexer-cli/src/__tests__/indexer/cost.test.ts index 329560bba..2532c68cd 100644 --- a/packages/indexer-cli/src/__tests__/indexer/cost.test.ts +++ b/packages/indexer-cli/src/__tests__/indexer/cost.test.ts @@ -214,30 +214,3 @@ describe('Indexer cost tests singleNetwork', () => { ) }) }) - -describe('Indexer cost tests multiNetworks', () => { - beforeAll(setupSingleNetwork) - afterAll(teardown) - beforeEach(seedCostModels) - afterEach(deleteFromAllTables) - - describe('Cost set...', () => { - cliTest( - 'Indexer cost set model deployment id - reject multinetwork mode', - [ - 'indexer', - 'cost', - 'set', - 'model', - 'QmXRpJW3qBuYaiBYHdhv8DF4bHDZhXBmh91MtrnhJfQ5Lk', - 'references/basic.agora', - ], - 'references/indexer-cost-model-deployment-multinetworks', - { - expectedExitCode: 1, - cwd: baseDir, - timeout: 10000, - }, - ) - }) -}) From 4b1726fa63bc1b3dd37ca6e3e0e0da9cda2d920e Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Wed, 19 Feb 2025 09:08:50 -0800 Subject: [PATCH 04/11] agent, common: address review comments on removal of multinetwork --- packages/indexer-agent/src/agent.ts | 21 ++++++------------- .../src/indexer-management/actions.ts | 1 + .../resolvers/allocations.ts | 4 ++-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 373c68c10..f3caa6908 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -184,13 +184,13 @@ export class Agent { } reconciliationLoop() { + const { network, operator } = this.networkAndOperator const requestIntervalSmall = this.pollingInterval const requestIntervalLarge = this.pollingInterval * 5 const logger = this.logger.child({ component: 'ReconciliationLoop' }) const currentEpochNumber: Eventual = sequentialTimerMap( { logger, milliseconds: requestIntervalLarge }, async () => { - const { network } = this.networkAndOperator logger.trace('Fetching current epoch number', { protocolNetwork: network.specification.networkIdentifier, }) @@ -202,8 +202,6 @@ export class Agent { }, ) - const { network } = this.networkAndOperator - const maxAllocationEpochs: Eventual = sequentialTimerMap( { logger, milliseconds: requestIntervalLarge }, async () => { @@ -222,7 +220,6 @@ export class Agent { sequentialTimerMap( { logger, milliseconds: requestIntervalSmall }, async () => { - const { network, operator } = this.networkAndOperator logger.trace('Fetching indexing rules', { protocolNetwork: network.specification.networkIdentifier, }) @@ -260,7 +257,6 @@ export class Agent { sequentialTimerMap( { logger, milliseconds: requestIntervalLarge }, async () => { - const { network } = this.networkAndOperator if ( this.deploymentManagement === DeploymentManagementMode.AUTO || network.networkMonitor.poiDisputeMonitoringEnabled() @@ -290,7 +286,6 @@ export class Agent { sequentialTimerMap( { logger, milliseconds: requestIntervalSmall }, async () => { - const { network } = this.networkAndOperator logger.trace('Fetching network deployments', { protocolNetwork: network.specification.networkIdentifier, }) @@ -367,7 +362,6 @@ export class Agent { const activeAllocations: Eventual = sequentialTimerMap( { logger, milliseconds: requestIntervalSmall }, async () => { - const { network } = this.networkAndOperator logger.trace('Fetching active allocations', { protocolNetwork: network.specification.networkIdentifier, }) @@ -389,13 +383,12 @@ export class Agent { }).tryMap( // eslint-disable-next-line @typescript-eslint/no-unused-vars async ({ activeAllocations: _, currentEpochNumber }) => { - const { network } = this.networkAndOperator - const allocationsByNetwork = + const recentlyClosedAllocations = await network.networkMonitor.recentlyClosedAllocations( currentEpochNumber, 1, ) - return Object.values(allocationsByNetwork).flat() + return Object.values(recentlyClosedAllocations).flat() }, { onError: () => @@ -410,7 +403,6 @@ export class Agent { activeDeployments, }).tryMap( async ({ currentEpochNumber, activeDeployments }) => { - const { network } = this.networkAndOperator logger.trace('Fetching disputable allocations', { protocolNetwork: network.specification.networkIdentifier, currentEpochNumber, @@ -455,7 +447,6 @@ export class Agent { }) try { - const { network, operator } = this.networkAndOperator const disputableEpochs = currentEpochNumber - network.specification.indexerOptions.poiDisputableEpochs @@ -918,7 +909,7 @@ export class Agent { // Network Subgraph is NEVER allocated towards // -------------------------------------------------------------------------------- const { network, operator } = this.networkAndOperator - let validatedAllocationDecisions = allocationDecisions + let validatedAllocationDecisions = [...allocationDecisions] if ( network.specification.indexerOptions.allocationManagementMode === @@ -940,12 +931,12 @@ export class Agent { networkSubgraphDeployment && !network.specification.indexerOptions.allocateOnNetworkSubgraph ) { - const networkSubgraphIndex = allocationDecisions.findIndex( + const networkSubgraphIndex = validatedAllocationDecisions.findIndex( decision => decision.deployment.bytes32 == networkSubgraphDeployment.id.bytes32, ) if (networkSubgraphIndex >= 0) { - allocationDecisions[networkSubgraphIndex].toAllocate = false + validatedAllocationDecisions[networkSubgraphIndex].toAllocate = false } } } diff --git a/packages/indexer-common/src/indexer-management/actions.ts b/packages/indexer-common/src/indexer-management/actions.ts index 9eebab11a..555c80b58 100644 --- a/packages/indexer-common/src/indexer-management/actions.ts +++ b/packages/indexer-common/src/indexer-management/actions.ts @@ -146,6 +146,7 @@ export class ActionManager { const networkLogger = logger.child({ protocolNetwork: this.network.specification.networkIdentifier, indexer: this.network.specification.indexerOptions.address, + operator: this.network.transactionManager.wallet.address, }) if (await this.batchReady(approvedActions, this.network, networkLogger)) { diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index b412beffb..3af7e6f8f 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -354,14 +354,14 @@ export default { protocolNetwork: network.specification.networkIdentifier, } - const allocationsByNetwork = await queryAllocations( + const allocationsResult = await queryAllocations( logger, networkSubgraph, variables, context, ) - return Object.values(allocationsByNetwork).flat() + return Object.values(allocationsResult).flat() }, createAllocation: async ( From 860aafb459be6d5085530a0a4dfc0da9d3eca850 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Thu, 13 Feb 2025 11:57:36 -0800 Subject: [PATCH 05/11] common, agent: use NetworksRegistry instead of hard-coded networks list --- packages/indexer-agent/src/commands/start.ts | 2 + packages/indexer-common/package.json | 1 + .../src/indexer-management/types.ts | 103 +++++++++++++----- yarn.lock | 5 + 4 files changed, 82 insertions(+), 29 deletions(-) diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index a4453f3de..8ffd97fbf 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -11,6 +11,7 @@ import { SubgraphDeploymentID, } from '@graphprotocol/common-ts' import { + common_init, createIndexerManagementClient, createIndexerManagementServer, defineIndexerManagementModels, @@ -467,6 +468,7 @@ export async function run( networkSpecification: spec.NetworkSpecification, logger: Logger, ): Promise { + await common_init(logger) // -------------------------------------------------------------------------------- // * Configure event listeners for unhandled promise rejections and uncaught // exceptions. diff --git a/packages/indexer-common/package.json b/packages/indexer-common/package.json index 76cd5b00c..22a566d75 100644 --- a/packages/indexer-common/package.json +++ b/packages/indexer-common/package.json @@ -22,6 +22,7 @@ "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo" }, "dependencies": { + "@pinax/graph-networks-registry": "0.6.7", "@graphprotocol/common-ts": "2.0.11", "@graphprotocol/cost-model": "0.1.18", "@semiotic-labs/tap-contracts-bindings": "^1.2.1", diff --git a/packages/indexer-common/src/indexer-management/types.ts b/packages/indexer-common/src/indexer-management/types.ts index cf801f4cb..7b4ec8c59 100644 --- a/packages/indexer-common/src/indexer-management/types.ts +++ b/packages/indexer-common/src/indexer-management/types.ts @@ -9,6 +9,14 @@ import { Allocation } from '../allocations' import { GraphNode } from '../graph-node' import { SubgraphDeployment } from '../types' +/* eslint-disable @typescript-eslint/no-explicit-any */ +let registry: any +async function initializeNetworksRegistry() { + // Dynamically import NetworksRegistry + const { NetworksRegistry } = await import('@pinax/graph-networks-registry') + registry = await NetworksRegistry.fromLatestVersion() +} + export interface CreateAllocationResult { actionID: number type: 'allocate' @@ -166,7 +174,34 @@ export function epochElapsedBlocks(networkEpoch: NetworkEpoch): number { return networkEpoch.startBlockNumber - networkEpoch.latestBlock } -export const Caip2ByChainAlias: { [key: string]: string } = { +// Construct Caip2ByChainId from the registry data, keeping the manual +// overrides for backward compatibility +const caip2ByChainId: { [key: number]: string } = { + 1337: 'eip155:1337', + 1: 'eip155:1', + 5: 'eip155:5', + 100: 'eip155:100', + 42161: 'eip155:42161', + 421613: 'eip155:421613', + 43114: 'eip155:43114', + 137: 'eip155:137', + 42220: 'eip155:42220', + 10: 'eip155:10', + 250: 'eip155:250', + 11155111: 'eip155:11155111', + 421614: 'eip155:421614', + 56: 'eip155:56', + 59144: 'eip155:59144', + 534352: 'eip155:534352', + 8453: 'eip155:8453', + 1284: 'eip155:1284', + 122: 'eip155:122', + 81457: 'eip155:81457', + 288: 'eip155:288', + 56288: 'eip155:56288', +} + +const caip2ByChainAlias: { [key: string]: string } = { mainnet: 'eip155:1', goerli: 'eip155:5', gnosis: 'eip155:100', @@ -191,29 +226,39 @@ export const Caip2ByChainAlias: { [key: string]: string } = { 'boba-bnb': 'eip155:56288', } -export const Caip2ByChainId: { [key: number]: string } = { - 1: 'eip155:1', - 5: 'eip155:5', - 100: 'eip155:100', - 1337: 'eip155:1337', - 42161: 'eip155:42161', - 421613: 'eip155:421613', - 43114: 'eip155:43114', - 137: 'eip155:137', - 42220: 'eip155:42220', - 10: 'eip155:10', - 250: 'eip155:250', - 11155111: 'eip155:11155111', - 421614: 'eip155:421614', - 56: 'eip155:56', - 59144: 'eip155:59144', - 534352: 'eip155:534352', - 8453: 'eip155:8453', - 1284: 'eip155:1284', - 122: 'eip155:122', - 81457: 'eip155:81457', - 288: 'eip155:288', - 56288: 'eip155:56288', +async function buildCaip2MappingsFromRegistry() { + const networks = registry.networks + + for (const network of networks) { + if (!network.aliases) { + continue + } + for (const alias of network.aliases) { + caip2ByChainAlias[alias] = network.caip2Id + } + const chainId = parseInt(network.caip2Id.split(':')[1]) + if ( + typeof chainId === 'number' && + !isNaN(chainId) && + !caip2ByChainId[chainId] // if we manually set an alias don't overwrite it + ) { + caip2ByChainId[+chainId] = network.caip2Id + } + } +} + +/** + * Unified async initialization needed for the common module. + * This function should be called once when an application starts. + * Needed to fetch & construct lookups for the networks registry. + */ +export async function common_init(logger: Logger) { + await initializeNetworksRegistry() + await buildCaip2MappingsFromRegistry() + logger.debug('Networks Registry loaded', { + caip2ByChainAlias, + caip2ByChainId, + }) } /// Unified entrypoint to resolve CAIP ID based either on chain aliases (strings) @@ -221,7 +266,7 @@ export const Caip2ByChainId: { [key: number]: string } = { export function resolveChainId(key: number | string): string { if (typeof key === 'number' || !isNaN(+key)) { // If key is a number, then it must be a `chainId` - const chainId = Caip2ByChainId[+key] + const chainId = caip2ByChainId[+key] if (chainId !== undefined) { return chainId } @@ -229,9 +274,9 @@ export function resolveChainId(key: number | string): string { const splitKey = key.split(':') let chainId if (splitKey.length === 2) { - chainId = Caip2ByChainId[+splitKey[1]] + chainId = caip2ByChainId[+splitKey[1]] } else { - chainId = Caip2ByChainAlias[key] + chainId = caip2ByChainAlias[key] } if (chainId !== undefined) { return chainId @@ -241,8 +286,8 @@ export function resolveChainId(key: number | string): string { } export function resolveChainAlias(id: string): string { - const aliasMatches = Object.keys(Caip2ByChainAlias).filter( - (name) => Caip2ByChainAlias[name] == id, + const aliasMatches = Object.keys(caip2ByChainAlias).filter( + (name) => caip2ByChainAlias[name] == id, ) if (aliasMatches.length === 1) { return aliasMatches[0] diff --git a/yarn.lock b/yarn.lock index 3443e3843..a54c11083 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2221,6 +2221,11 @@ node-addon-api "^3.2.1" node-gyp-build "^4.3.0" +"@pinax/graph-networks-registry@0.6.7": + version "0.6.7" + resolved "https://registry.yarnpkg.com/@pinax/graph-networks-registry/-/graph-networks-registry-0.6.7.tgz#ceb994f3b31e2943b9c9d9b09dd86eb00d067c0e" + integrity sha512-xogeCEZ50XRMxpBwE3TZjJ8RCO8Guv39gDRrrKtlpDEDEMLm0MzD3A0SQObgj7aF7qTZNRTWzsuvQdxgzw25wQ== + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz" From 5b91f799e617a2328731ddb6b292dbe00c944db5 Mon Sep 17 00:00:00 2001 From: Andrew Clews Date: Mon, 20 Jan 2025 12:29:03 -0500 Subject: [PATCH 06/11] Update feature-support-matrix.md Updated row 51 with the latest council snapshot --- docs/feature-support-matrix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/feature-support-matrix.md b/docs/feature-support-matrix.md index ca27b5961..592304a3f 100644 --- a/docs/feature-support-matrix.md +++ b/docs/feature-support-matrix.md @@ -48,7 +48,7 @@ graph-node: >=0.35.0 <=0.36.1 ### Latest Council snapshot -[GGP-0043: Updated Feature Matrix Support (base, bsc, scroll, linea)](https://snapshot.org/#/council.graphprotocol.eth/proposal/0x4226f917346416b76f77369181881bebbc9b4e3f66e9681e5fc4e56d7460eba7) +[GGP-0050 Updated Feature Matrix Support (zora, mode, moonbeam)](https://snapshot.org/#/s:council.graphprotocol.eth/proposal/0x7c1b0eaa299a24ba23f76d86d85b903ac8e8457db3656531e7bd5cee80c20146) ### Other notes From 07f96984e5c442c0403b91f81cdd64a95d1c5e94 Mon Sep 17 00:00:00 2001 From: Andrew-Clews Date: Thu, 12 Dec 2024 13:24:54 -0500 Subject: [PATCH 07/11] docs & common: add support for zora, mode, moonbeam docs & common: add support for zora, mode, moonbeam Updated feature support matrix and types.ts files. --- docs/feature-support-matrix.md | 3 +++ packages/indexer-common/src/indexer-management/types.ts | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/docs/feature-support-matrix.md b/docs/feature-support-matrix.md index 592304a3f..edcc93461 100644 --- a/docs/feature-support-matrix.md +++ b/docs/feature-support-matrix.md @@ -31,6 +31,9 @@ The matrix below reflects the canonical Council-ratified version. As outlined in | eip155:81457 | blast-mainnet | Yes | Yes | Yes | Yes | Yes | | eip155:288 | boba | Yes | Yes | Yes | Yes | Yes | | eip155:56288 | boba-bnb | Yes | Yes | Yes | Yes | Yes | +| eip155:7777777 | zora | Yes | Yes | Yes | Yes | Yes | +| eip155:34443 | mode | Yes | Yes | Yes | Yes | Yes | +| eip155:1284 | moonbeam | Yes | Yes | Yes | Yes | Yes | | **Data Source Features** | | | | | | | | ipfs.cat in mappings | | Yes | Yes | No | No | No | | ENS | | Yes | Yes | Yes | Yes | Yes | diff --git a/packages/indexer-common/src/indexer-management/types.ts b/packages/indexer-common/src/indexer-management/types.ts index 7b4ec8c59..b4fb2b36e 100644 --- a/packages/indexer-common/src/indexer-management/types.ts +++ b/packages/indexer-common/src/indexer-management/types.ts @@ -199,6 +199,8 @@ const caip2ByChainId: { [key: number]: string } = { 81457: 'eip155:81457', 288: 'eip155:288', 56288: 'eip155:56288', + 7777777: 'eip155:7777777', + 34443: 'eip155:34443', } const caip2ByChainAlias: { [key: string]: string } = { @@ -224,6 +226,8 @@ const caip2ByChainAlias: { [key: string]: string } = { 'blast-mainnet': 'eip155:81457', boba: 'eip155:288', 'boba-bnb': 'eip155:56288', + zora: 'eip155:7777777', + mode: 'eip155:34443', } async function buildCaip2MappingsFromRegistry() { From eb1a786c9024ac4e7cfb5b320f27e02c212f992d Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Fri, 21 Feb 2025 11:26:11 -0800 Subject: [PATCH 08/11] common: also add chain aliases stripped of their -mainnet suffix --- packages/indexer-common/src/indexer-management/types.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/indexer-common/src/indexer-management/types.ts b/packages/indexer-common/src/indexer-management/types.ts index b4fb2b36e..69719eda5 100644 --- a/packages/indexer-common/src/indexer-management/types.ts +++ b/packages/indexer-common/src/indexer-management/types.ts @@ -239,6 +239,10 @@ async function buildCaip2MappingsFromRegistry() { } for (const alias of network.aliases) { caip2ByChainAlias[alias] = network.caip2Id + if (alias.endsWith('-mainnet')) { + const aliasWithoutSuffix = alias.replace('-mainnet', '') + caip2ByChainAlias[aliasWithoutSuffix] = network.caip2Id + } } const chainId = parseInt(network.caip2Id.split(':')[1]) if ( From 302f6012809d9d13fc14df357757321d6adebdf3 Mon Sep 17 00:00:00 2001 From: Andrew Clews Date: Fri, 21 Feb 2025 13:56:33 -0500 Subject: [PATCH 09/11] docs & common: add support for rootstock, remove cosmos Updated feature support matrix to include rootstock and removed cosmos --- docs/feature-support-matrix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/feature-support-matrix.md b/docs/feature-support-matrix.md index edcc93461..61d033e9f 100644 --- a/docs/feature-support-matrix.md +++ b/docs/feature-support-matrix.md @@ -15,7 +15,6 @@ The matrix below reflects the canonical Council-ratified version. As outlined in | eip155:1 | mainnet | Yes | No | Yes | Yes | Yes | | eip155:100 | gnosis | Yes | Yes | Yes | Yes | Yes | | near:\* | \* | Yes | Yes | No | No | No | -| cosmos:\* | \* | Yes | Yes | No | No | No | | arweave:\* | \* | Yes | Yes | No | No | No | | eip155:42161 | artbitrum-one | Yes | Yes | Yes | Yes | Yes | | eip155:42220 | celo | Yes | Yes | Yes | Yes | Yes | @@ -34,6 +33,7 @@ The matrix below reflects the canonical Council-ratified version. As outlined in | eip155:7777777 | zora | Yes | Yes | Yes | Yes | Yes | | eip155:34443 | mode | Yes | Yes | Yes | Yes | Yes | | eip155:1284 | moonbeam | Yes | Yes | Yes | Yes | Yes | +| eip155:30 | rootstock | Yes | Yes | Yes | Yes | Yes | | **Data Source Features** | | | | | | | | ipfs.cat in mappings | | Yes | Yes | No | No | No | | ENS | | Yes | Yes | Yes | Yes | Yes | From e9900f8319fe9e74abf7a104b74e8b4e3cca03a9 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Mon, 24 Feb 2025 14:38:37 -0300 Subject: [PATCH 10/11] chore: add local testing script --- .env.example | 5 +++++ README.md | 35 +++++++++++++++++++++++++++++++++++ scripts/run-tests.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 .env.example create mode 100644 scripts/run-tests.sh diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..bbb19be81 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# Arbitrum Sepolia testnet JSON-RPC provider URL for testing (e.g., Infura, Alchemy, etc.) +INDEXER_TEST_JRPC_PROVIDER_URL=https://arbitrum-sepolia.infura.io/v3/your-project-id + +# The Graph API key for querying subgraphs (from The Graph Studio) +INDEXER_TEST_API_KEY=your-graph-api-key-here diff --git a/README.md b/README.md index 5cf7a822b..9da8f4318 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,41 @@ Creating a new release involves the following steps: yarn release ``` +## Running tests locally + +To run the tests locally, you'll need: +1. Docker installed and running +2. Node.js and Yarn +3. An Arbitrum Sepolia testnet RPC provider (e.g., Infura, Alchemy) +4. An API key from The Graph Studio for querying subgraphs + +### Setup + +1. Create a `.env` file in the root directory with your credentials. You can copy the example file as a template: +```sh +cp .env.example .env +``` + +Then edit `.env` with your credentials: +```plaintext +# Your Arbitrum Sepolia testnet RPC endpoint +INDEXER_TEST_JRPC_PROVIDER_URL=https://sepolia.infura.io/v3/your-project-id + +# Your API key from The Graph Studio (https://thegraph.com/studio/) +INDEXER_TEST_API_KEY=your-graph-api-key-here +``` + +2. Run the tests: +```sh +bash scripts/run-tests.sh +``` + +The script will: +- Start a PostgreSQL container with the required test configuration +- Load your credentials from the `.env` file +- Run the test suite +- Clean up the PostgreSQL container when done + # Copyright Copyright © 2020-2021 The Graph Foundation diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100644 index 000000000..a2849cb93 --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Cleanup function to stop and remove the container +cleanup() { + echo "Cleaning up..." + docker stop indexer-test-db >/dev/null 2>&1 + docker rm indexer-test-db >/dev/null 2>&1 +} + +# Register the cleanup function to run on script exit +trap cleanup EXIT + +# Start PostgreSQL container with the same configuration as CI +docker run -d \ + --name indexer-test-db \ + -e POSTGRES_DB=indexer_tests \ + -e POSTGRES_USER=testuser \ + -e POSTGRES_PASSWORD=testpass \ + -p 5432:5432 \ + postgres:13 + +# Wait for PostgreSQL to be ready +echo "Waiting for PostgreSQL to be ready..." +until docker exec indexer-test-db pg_isready > /dev/null 2>&1; do + sleep 1 +done +echo "PostgreSQL is ready!" + +# Load environment variables from .env file +if [ -f .env ]; then + echo "Loading .env file..." + source .env +else + echo "Warning: .env file not found" +fi + +# Run the tests +echo "Running tests..." +POSTGRES_TEST_HOST=localhost \ +POSTGRES_TEST_DATABASE=indexer_tests \ +POSTGRES_TEST_USERNAME=testuser \ +POSTGRES_TEST_PASSWORD=testpass \ +NODE_OPTIONS="--dns-result-order=ipv4first" \ +yarn test:ci From c1f990f4064bac0abae83a129fb9d65be9133348 Mon Sep 17 00:00:00 2001 From: Andrew Clews Date: Wed, 5 Mar 2025 13:39:02 -0500 Subject: [PATCH 11/11] docs & common: add support for Polygon zkEVM, Arbitrum Sepolia, zkSync Era Do not merge until council votes on associated snapshot vote. Currently planned for GGP 52. Request is to update the feature support matrix with 3 new chains: Polygon zkEVM, Arbitrum Sepolia, zkSync Era pending approval from the council. --- docs/feature-support-matrix.md | 77 ++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/docs/feature-support-matrix.md b/docs/feature-support-matrix.md index 61d033e9f..b4d6a8bbd 100644 --- a/docs/feature-support-matrix.md +++ b/docs/feature-support-matrix.md @@ -4,43 +4,46 @@ As described in [GIP-0008](https://snapshot.org/#/council.graphprotocol.eth/prop The matrix below reflects the canonical Council-ratified version. As outlined in GIP-00008, Council ratification is currently required for each update, which may happen at different stages of feature development and testing lifecycle. -| Subgraph Feature | Aliases | Implemented | Experimental | Query Arbitration | Indexing Arbitration | Indexing Rewards | -| -------------------------- | ------------- | ----------- | ------------ | ----------------- | -------------------- | ---------------- | -| **Core Features** | | | | | | | -| Full-text Search | | Yes | No | No | Yes | Yes | -| Non-Fatal Errors | | Yes | Yes | Yes | Yes | Yes | -| Grafting | | Yes | Yes | Yes | Yes | Yes | -| **Data Source Types** | | | | | | | -| eip155:\* | \* | Yes | No | No | No | No | -| eip155:1 | mainnet | Yes | No | Yes | Yes | Yes | -| eip155:100 | gnosis | Yes | Yes | Yes | Yes | Yes | -| near:\* | \* | Yes | Yes | No | No | No | -| arweave:\* | \* | Yes | Yes | No | No | No | -| eip155:42161 | artbitrum-one | Yes | Yes | Yes | Yes | Yes | -| eip155:42220 | celo | Yes | Yes | Yes | Yes | Yes | -| eip155:43114 | avalanche | Yes | Yes | Yes | Yes | Yes | -| eip155:250 | fantom | Yes | Yes | Yes | Yes | Yes | -| eip155:137 | polygon | Yes | Yes | Yes | Yes | Yes | -| eip155:10 | optimism | Yes | Yes | Yes | Yes | Yes | -| eip155:8453 | base | Yes | Yes | Yes | Yes | Yes | -| eip155:534352 | scroll | Yes | Yes | Yes | Yes | Yes | -| eip155:59144 | linea | Yes | Yes | Yes | Yes | Yes | -| eip155:56 | bsc | Yes | Yes | Yes | Yes | Yes | -| eip155:122 | fuse | Yes | Yes | Yes | Yes | Yes | -| eip155:81457 | blast-mainnet | Yes | Yes | Yes | Yes | Yes | -| eip155:288 | boba | Yes | Yes | Yes | Yes | Yes | -| eip155:56288 | boba-bnb | Yes | Yes | Yes | Yes | Yes | -| eip155:7777777 | zora | Yes | Yes | Yes | Yes | Yes | -| eip155:34443 | mode | Yes | Yes | Yes | Yes | Yes | -| eip155:1284 | moonbeam | Yes | Yes | Yes | Yes | Yes | -| eip155:30 | rootstock | Yes | Yes | Yes | Yes | Yes | -| **Data Source Features** | | | | | | | -| ipfs.cat in mappings | | Yes | Yes | No | No | No | -| ENS | | Yes | Yes | Yes | Yes | Yes | -| File data sources: Arweave | | Yes | Yes | No | Yes | Yes | -| File data sources: IPFS | | Yes | Yes | No | Yes | Yes | -| Substreams: mainnet | | Yes | Yes | Yes | Yes | Yes | -| Substreams: optimism | | Yes | Yes | Yes | Yes | Yes | +| Subgraph Feature | Aliases | Implemented | Experimental | Query Arbitration | Indexing Arbitration | Indexing Rewards | +| -------------------------- | ---------------- | ----------- | ------------ | ----------------- | -------------------- | ---------------- | +| **Core Features** | | | | | | | +| Full-text Search | | Yes | No | No | Yes | Yes | +| Non-Fatal Errors | | Yes | Yes | Yes | Yes | Yes | +| Grafting | | Yes | Yes | Yes | Yes | Yes | +| **Data Source Types** | | | | | | | +| eip155:\* | \* | Yes | No | No | No | No | +| eip155:1 | mainnet | Yes | No | Yes | Yes | Yes | +| eip155:100 | gnosis | Yes | Yes | Yes | Yes | Yes | +| near:\* | \* | Yes | Yes | No | No | No | +| arweave:\* | \* | Yes | Yes | No | No | No | +| eip155:42161 | artbitrum-one | Yes | Yes | Yes | Yes | Yes | +| eip155:42220 | celo | Yes | Yes | Yes | Yes | Yes | +| eip155:43114 | avalanche | Yes | Yes | Yes | Yes | Yes | +| eip155:250 | fantom | Yes | Yes | Yes | Yes | Yes | +| eip155:137 | polygon | Yes | Yes | Yes | Yes | Yes | +| eip155:10 | optimism | Yes | Yes | Yes | Yes | Yes | +| eip155:8453 | base | Yes | Yes | Yes | Yes | Yes | +| eip155:534352 | scroll | Yes | Yes | Yes | Yes | Yes | +| eip155:59144 | linea | Yes | Yes | Yes | Yes | Yes | +| eip155:56 | bsc | Yes | Yes | Yes | Yes | Yes | +| eip155:122 | fuse | Yes | Yes | Yes | Yes | Yes | +| eip155:81457 | blast-mainnet | Yes | Yes | Yes | Yes | Yes | +| eip155:288 | boba | Yes | Yes | Yes | Yes | Yes | +| eip155:56288 | boba-bnb | Yes | Yes | Yes | Yes | Yes | +| eip155:7777777 | zora | Yes | Yes | Yes | Yes | Yes | +| eip155:34443 | mode | Yes | Yes | Yes | Yes | Yes | +| eip155:1284 | moonbeam | Yes | Yes | Yes | Yes | Yes | +| eip155:30 | rootstock | Yes | Yes | Yes | Yes | Yes | +| eip155:1101 | polygon-zkevm | Yes | Yes | Yes | Yes | Yes | +| eip155:421614 | arbitrum-sepolia | Yes | Yes | Yes | Yes | Yes | +| eip155:324 | zksync-era | Yes | Yes | Yes | Yes | Yes | +| **Data Source Features** | | | | | | | +| ipfs.cat in mappings | | Yes | Yes | No | No | No | +| ENS | | Yes | Yes | Yes | Yes | Yes | +| File data sources: Arweave | | Yes | Yes | No | Yes | Yes | +| File data sources: IPFS | | Yes | Yes | No | Yes | Yes | +| Substreams: mainnet | | Yes | Yes | Yes | Yes | Yes | +| Substreams: optimism | | Yes | Yes | Yes | Yes | Yes | The accepted `graph-node` version range must always be specified; it always comprises the latest available version and the one immediately preceding it. The latest for the feature matrix above: