fixes for nodelist oos

turn off config patching, induce churn to the refute arrays
wait 4 min before syncing data

additional logging / log changes.  added a second spot in digest cycle to patch the nodeListHash after we applyNodeListChange

fix idea 2.  syncV2 already has the nodelist so we don't want to transform it anymore

more logs

fix in initRefuteCyclesForNode

a few more updates

adjust logs / turn off debug code

remove comment

WIP on itn4-1.16.4rc1

format fix

Author: afostr

src/logger/index.ts

@@ -120,6 +120,8 @@ export type LogFlags = {
   verboseNestedCounters: boolean // extra logging for nested counters
 
   node_rotation_debug: boolean // extra logging for node rotation math
+
+  p2pSyncDebug: boolean
 }
 
 export let logFlags: LogFlags = {
@@ -163,6 +165,8 @@ export let logFlags: LogFlags = {
   verboseNestedCounters: false,
 
   node_rotation_debug: false,
+
+  p2pSyncDebug: false,
 }
 
 const filePath1 = path.join(process.cwd(), 'data-logs', 'cycleRecords1.txt')

src/p2p/CycleChain.ts

@@ -81,6 +81,9 @@ export function validate(
   info('validate: prev.standbylist', prev.standbyNodeListHash)
   info('validate: next.standbylist', next.standbyNodeListHash)
 
+  info('validate: prev.nodelist', prev.nodeListHash)
+  info('validate: next.nodelist', next.nodeListHash)
+
   if (next.previous !== prevMarker) {
     info('validate: ERROR: next.previous !== prevMarker')
     return false

src/p2p/NodeList.ts

@@ -114,7 +114,12 @@ const isRefuteCyclesEnabled = (cycle: P2P.CycleCreatorTypes.CycleRecord | null)
 
 const initRefuteCyclesForNode = (node: P2P.NodeListTypes.Node, cycle: P2P.CycleCreatorTypes.CycleRecord | null) => {
   if (isRefuteCyclesEnabled(cycle)) {
-    node.refuteCycles = []
+    // This if check is a critical fix.  This is because initRefuteCyclesForNode gets called
+    // as part of addNode and addNode gets called in syncV2 after we download the nodelist and then 
+    // pass it to addNodes.   having the check here will make sure we dont wipe out existing
+    // data in refuteCycles that we just downloaded.
+    if(node.refuteCycles == null)
+      node.refuteCycles = []
   }
 }
 
@@ -391,6 +396,26 @@ export function updateNodes(
   cycle: P2P.CycleCreatorTypes.CycleRecord | null
 ) {
   for (const update of updates) updateNode(update, raiseEvents, cycle)
+
+  /* //LOCAL_OOS_TEST_SUPPORT
+  let stats = {init:0, push:0}
+  const refuteCyclesEnabled = isRefuteCyclesEnabled(cycle)
+  // every cycle mess with every node 
+  for (const node of activeByIdOrder){
+    if (refuteCyclesEnabled) {
+      if(node.refuteCycles == null){
+        node.refuteCycles = []
+        stats.init++
+      } else {
+        node.refuteCycles.push(cycle.counter)
+        stats.push++
+      }
+    }
+  }
+
+  info(`NodeList.updateNodes: ${updates.length} ${Utils.safeStringify(stats)} enableProblematicNodeRemoval: ${config.p2p.enableProblematicNodeRemoval} cycle: ${cycle?.counter} refuteCyclesEnabled: ${refuteCyclesEnabled} enableProblematicNodeRemovalOnCycle: ${config.p2p.enableProblematicNodeRemovalOnCycle}`) 
+  */
+
 }
 
 export function updateProblematicNodeTracking(cycle: P2P.CycleCreatorTypes.CycleRecord | null) {

src/p2p/Sync.ts

@@ -4,7 +4,7 @@ import util from 'util'
 import * as http from '../http'
 import { P2P } from '@shardus/types'
 import { reversed, validateTypes } from '../utils'
-import { config, logger, network } from './Context'
+import { config, crypto, logger, network } from './Context'
 import * as Archivers from './Archivers'
 import * as CycleChain from './CycleChain'
 import * as CycleCreator from './CycleCreator'
@@ -366,11 +366,11 @@ export function digestCycle(cycle: P2P.CycleCreatorTypes.CycleRecord, source: st
     // standby list, but not with the validator and archivers lists
 
     const newNodeListHash = NodeList.computeNewNodeListHash()
-    if (source === 'syncV2' && newNodeListHash !== cycle.nodeListHash) warn(`sync:digestCycle source: ${source} cycle: ${cycle.counter} patching nodelisthash ${cycle.nodeListHash} -> ${newNodeListHash}`)
+    if (newNodeListHash !== cycle.nodeListHash) warn(`sync:digestCycle source: ${source} cycle: ${cycle.counter} patching nodelisthash ${cycle.nodeListHash} -> ${newNodeListHash}`)
     cycle.nodeListHash = newNodeListHash
 
     const newArchiverListHash = Archivers.computeNewArchiverListHash()
-    if (source === 'syncV2' && newArchiverListHash !== cycle.archiverListHash) warn(`sync:digestCycle source: ${source} cycle: ${cycle.counter} patching archiverlisthash ${cycle.archiverListHash} -> ${newArchiverListHash}`)
+    if (newArchiverListHash !== cycle.archiverListHash) warn(`sync:digestCycle source: ${source} cycle: ${cycle.counter} patching archiverlisthash ${cycle.archiverListHash} -> ${newArchiverListHash}`)
     cycle.archiverListHash = newArchiverListHash
 
     // for join v2, also get the standby node list hash
@@ -419,9 +419,16 @@ export function digestCycle(cycle: P2P.CycleCreatorTypes.CycleRecord, source: st
       )} CycleCreator.currentCycle: ${CycleCreator.currentCycle}`
     )
 
+  
   const changes = parse(cycle)
+
   applyNodeListChange(changes, true, cycle)
 
+  if (logFlags.important_as_error) {
+    const newNodeListHash = crypto.hash(NodeList.byJoinOrder) //computeNewNodeListHash not safe due to side effects
+    warn(`sync:digestCycle after applyNodeListChange source: ${source} cycle: ${cycle.counter} prev nodelisthash ${cycle.nodeListHash} next ${newNodeListHash}`)
+  }
+
   // for join v2, also add any new standby nodes to the standy node list
   // and remove any standby nodes that have unjoined.
   if (config.p2p.useJoinProtocolV2) {
@@ -446,7 +453,10 @@ export function digestCycle(cycle: P2P.CycleCreatorTypes.CycleRecord, source: st
   }
 
   CycleChain.append(cycle)
-  info(`digestCycle: marker of cycle${cycle.counter} from ${source} after digest is ${CycleChain.computeCycleMarker(cycle)}`)
+  const digestedCycleMarker = CycleChain.computeCycleMarker(cycle)
+  info(`digestCycle: marker of cycle${cycle.counter} from ${source} after digest is ${digestedCycleMarker}`)
+
+  /* prettier-ignore */ if (logFlags.important_as_error) info(`digestCycle: cycle: ${Utils.safeStringify(cycle)}`)
 
   // TODO: This seems like a possible location to inetvene if our node
   // is getting far behind on what it thinks the current cycle is

src/p2p/SyncV2/queries.ts

@@ -156,6 +156,8 @@ export function getCycleDataFromNode(
   node: ActiveNode,
   expectedMarker: hexstring
 ): ResultAsync<CycleRecord, Error> {
+  info(`getCycleDataFromNode: expectedMarker: ${expectedMarker}`)
+  
   return attemptSimpleFetch(node, 'cycle-by-marker', {
     marker: expectedMarker,
   })
@@ -166,6 +168,8 @@ export function getValidatorListFromNode(
   node: ActiveNode,
   expectedHash: hexstring
 ): ResultAsync<Validator[], Error> {
+  info(`getValidatorListFromNode: expectedHash: ${expectedHash}`)
+
   return attemptSimpleFetch(
     node,
     'validator-list',
@@ -181,6 +185,8 @@ export function getArchiverListFromNode(
   node: ActiveNode,
   expectedHash: hexstring
 ): ResultAsync<Archiver[], Error> {
+  info(`getArchiverListFromNode: expectedHash: ${expectedHash}`)
+
   return attemptSimpleFetch(node, 'archiver-list', {
     hash: expectedHash,
   })
@@ -191,6 +197,8 @@ export function getStandbyNodeListFromNode(
   node: ActiveNode,
   expectedHash: hexstring
 ): ResultAsync<JoinRequest[], Error> {
+  info(`getStandbyNodeListFromNode: expectedHash: ${expectedHash}`)
+
   return attemptSimpleFetch(
     node,
     'standby-list',
@@ -206,6 +214,8 @@ export function getTxListFromNode(
   node: ActiveNode,
   expectedHash: hexstring
 ): ResultAsync<{ hash: string; tx: P2P.ServiceQueueTypes.AddNetworkTx }[], Error> {
+  info(`getTxListFromNode: expectedHash: ${expectedHash}`)
+
   return attemptSimpleFetch(
     node,
     'tx-list',
@@ -215,3 +225,8 @@ export function getTxListFromNode(
     10000 //TODO need to make this scale when there could be millions of entries
   )
 }
+
+function info(...msg) {
+  const entry = `SyncV2: ${msg.join(' ')}`
+  p2pLogger.info(entry)
+}

src/p2p/SyncV2/verify.ts

@@ -9,6 +9,8 @@ import { HashableObject } from '../../crypto'
 import { crypto } from '../Context'
 import { makeCycleMarker } from '../CycleCreator'
 import { Utils } from '@shardus/types'
+import { p2pLogger } from './queries'
+import { logFlags } from '../../logger'
 
 /**
  * Verifies if the hash of a given object matches the expected hash.
@@ -39,6 +41,8 @@ export function verifyValidatorList(
   validatorList: P2P.NodeListTypes.Node[],
   expectedHash: hexstring
 ): Result<boolean, Error> {
+  
+  if(logFlags.p2pSyncDebug) info(`verifyValidatorList ${expectedHash}  ${Utils.safeStringify(validatorList)} `);
   return verify(validatorList, expectedHash, 'validator list')
 }
 
@@ -76,3 +80,8 @@ export function verifyTxList(
 
   return ok(true)
 }
+
+function info(...msg) {
+  const entry = `SyncV2: ${msg.join(' ')}`
+  p2pLogger.info(entry)
+}

src/shardus/index.ts

@@ -772,7 +772,14 @@ class Shardus extends EventEmitter {
         // todo hook this up later cant deal with it now.
         // await this.storage.deleteOldDBPath()
 
+        /* // LOCAL_OOS_TEST_SUPPORT  not for production
+          this.mainLogger.info('sync-p2p synced waiting 4 min')
+          await utils.sleep(240000) //do not release this helps us have a chance
+          //to query /config before the node syncs data
+        */
+        this.mainLogger.info('sync-syncAppData')
         await this.syncAppData()
+
       }
     })
     Self.emitter.on('restore', async (cycleNumber: number) => {