Skip to content

[tmpnet] Delegate writing of the flag file to the runtime #3897

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 28, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/antithesis/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
}

// Apply configuration appropriate to a test network
for k, v := range tmpnet.DefaultTestFlags() {
for k, v := range tmpnet.DefaultTmpnetFlags() {
env[k] = v
}

Expand Down
20 changes: 12 additions & 8 deletions tests/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,22 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
require.NoError(tc, err)

upgradeBase64 := base64.StdEncoding.EncodeToString(upgradeJSON)

defaultFlags := tmpnet.FlagsMap{
config.UpgradeFileContentKey: upgradeBase64,
// Ensure a min stake duration compatible with testing staking logic
config.MinStakeDurationKey: "1s",
}
defaultFlags.SetDefaults(tmpnet.DefaultE2EFlags())
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The e2e flags are now expected to be set on the network rather than applied automatically by tmpnet.


return e2e.NewTestEnvironment(
tc,
flagVars,
&tmpnet.Network{
Owner: flagVars.NetworkOwner(),
DefaultFlags: tmpnet.FlagsMap{
config.UpgradeFileContentKey: upgradeBase64,
// Ensure a min stake duration compatible with testing staking logic
config.MinStakeDurationKey: tmpnet.DefaultMinStakeDuration,
},
Nodes: nodes,
Subnets: subnets,
Owner: flagVars.NetworkOwner(),
DefaultFlags: defaultFlags,
Nodes: nodes,
Subnets: subnets,
},
).Marshal()
}, func(envBytes []byte) {
Expand Down
24 changes: 6 additions & 18 deletions tests/fixture/tmpnet/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ const (
defaultConfigFilename = "config.json"
)

// Flags appropriate for networks used for all types of testing.
func DefaultTestFlags() FlagsMap {
// Flags suggested for temporary networks. Applied by default.
func DefaultTmpnetFlags() FlagsMap {
return FlagsMap{
config.NetworkPeerListPullGossipFreqKey: "250ms",
config.NetworkMaxReconnectDelayKey: "1s",
Expand All @@ -46,26 +46,14 @@ func DefaultTestFlags() FlagsMap {
}
}

// Flags appropriate for tmpnet networks.
func DefaultTmpnetFlags() FlagsMap {
// Supply only non-default configuration to ensure that default values will be used.
flags := FlagsMap{
// Default to dynamic port allocation
config.HTTPPortKey: "0",
config.StakingPortKey: "0",
// Specific to tmpnet deployment
config.PublicIPKey: "127.0.0.1",
config.HTTPHostKey: "127.0.0.1",
config.StakingHostKey: "127.0.0.1",
config.LogDisplayLevelKey: logging.Off.String(), // Display logging not needed since nodes run headless
config.LogLevelKey: logging.Debug.String(),
// Specific to e2e testing
// Flags suggested for e2e testing
func DefaultE2EFlags() FlagsMap {
return FlagsMap{
config.ProposerVMUseCurrentHeightKey: "true",
// Reducing this from the 1s default speeds up tx acceptance
config.ProposerVMMinBlockDelayKey: "0s",
config.LogLevelKey: logging.Debug.String(),
}
flags.SetDefaults(DefaultTestFlags())
return flags
}

// A set of chain configurations appropriate for testing.
Expand Down
93 changes: 12 additions & 81 deletions tests/fixture/tmpnet/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"net/netip"
"os"
"os/exec"
Expand Down Expand Up @@ -65,7 +64,8 @@ var (
// TODO(marun) Remove when subnet-evm configures the genesis with this key.
HardhatKey *secp256k1.PrivateKey

errInsufficientNodes = errors.New("at least one node is required")
errInsufficientNodes = errors.New("at least one node is required")
errMissingRuntimeConfig = errors.New("DefaultRuntimeConfig must not be empty")
)

func init() {
Expand Down Expand Up @@ -253,6 +253,11 @@ func (n *Network) EnsureDefaultConfig(log logging.Logger) error {
}
}

emptyRuntime := NodeRuntimeConfig{}
if n.DefaultRuntimeConfig == emptyRuntime {
return errMissingRuntimeConfig
}

return nil
}

Expand Down Expand Up @@ -471,10 +476,6 @@ func (n *Network) StartNode(ctx context.Context, node *Node) error {
return err
}

if err := n.writeNodeFlags(node); err != nil {
return fmt.Errorf("writing node flags: %w", err)
}

if err := node.Start(ctx); err != nil {
// Attempt to stop an unhealthy node to provide some assurance to the caller
// that an error condition will not result in a lingering process.
Expand Down Expand Up @@ -762,10 +763,11 @@ func (n *Network) GetNodeURIs() []NodeURI {
return GetNodeURIs(n.Nodes)
}

// Retrieves bootstrap IPs and IDs for all nodes except the skipped one (this supports
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This cleanup opportunity was discovered in the process of updating how flags were written.

// collecting the bootstrap details for restarting a node).
// Retrieves bootstrap IPs and IDs for all non-ephemeral nodes except the skipped one
// (this supports collecting the bootstrap details for restarting a node).
//
// For consumption outside of avalanchego. Needs to be kept exported.
func (n *Network) GetBootstrapIPsAndIDs(skippedNode *Node) ([]string, []string, error) {
func (n *Network) GetBootstrapIPsAndIDs(skippedNode *Node) ([]string, []string) {
bootstrapIPs := []string{}
bootstrapIDs := []string{}
for _, node := range n.Nodes {
Expand All @@ -786,7 +788,7 @@ func (n *Network) GetBootstrapIPsAndIDs(skippedNode *Node) ([]string, []string,
bootstrapIDs = append(bootstrapIDs, node.NodeID.String())
}

return bootstrapIPs, bootstrapIDs, nil
return bootstrapIPs, bootstrapIDs
}

// GetNetworkID returns the effective ID of the network. If the network
Expand Down Expand Up @@ -879,77 +881,6 @@ func (n *Network) GetChainConfigContent() (string, error) {
return base64.StdEncoding.EncodeToString(marshaledConfigs), nil
}

// writeNodeFlags determines the set of flags that should be used to
// start the given node and writes them to a file in the node path.
func (n *Network) writeNodeFlags(node *Node) error {
flags := maps.Clone(node.Flags)

// Convert the network id to a string to ensure consistency in JSON round-tripping.
flags.SetDefault(config.NetworkNameKey, strconv.FormatUint(uint64(n.GetNetworkID()), 10))

// Set the bootstrap configuration
bootstrapIPs, bootstrapIDs, err := n.GetBootstrapIPsAndIDs(node)
if err != nil {
return fmt.Errorf("failed to determine bootstrap configuration: %w", err)
}
flags.SetDefault(config.BootstrapIDsKey, strings.Join(bootstrapIDs, ","))
flags.SetDefault(config.BootstrapIPsKey, strings.Join(bootstrapIPs, ","))

// TODO(marun) Maybe avoid computing content flags for each node start?

if n.Genesis != nil {
genesisFileContent, err := n.GetGenesisFileContent()
if err != nil {
return fmt.Errorf("failed to get genesis file content: %w", err)
}
flags.SetDefault(config.GenesisFileContentKey, genesisFileContent)

isSingleNodeNetwork := (len(n.Nodes) == 1 && len(n.Genesis.InitialStakers) == 1)
if isSingleNodeNetwork {
n.log.Info("defaulting to sybil protection disabled to enable a single-node network to start")
flags.SetDefault(config.SybilProtectionEnabledKey, "false")
}
}

subnetConfigContent, err := n.GetSubnetConfigContent()
if err != nil {
return fmt.Errorf("failed to get subnet config content: %w", err)
}
if len(subnetConfigContent) > 0 {
flags.SetDefault(config.SubnetConfigContentKey, subnetConfigContent)
}

chainConfigContent, err := n.GetChainConfigContent()
if err != nil {
return fmt.Errorf("failed to get chain config content: %w", err)
}
if len(chainConfigContent) > 0 {
flags.SetDefault(config.ChainConfigContentKey, chainConfigContent)
}

// Only configure the plugin dir with a non-empty value to ensure the use of
// the default value (`[datadir]/plugins`) when no plugin dir is configured.
processConfig := node.getRuntimeConfig().Process
if processConfig != nil {
if len(processConfig.PluginDir) > 0 {
// Ensure the plugin directory exists or the node will fail to start
if err := os.MkdirAll(processConfig.PluginDir, perms.ReadWriteExecute); err != nil {
return fmt.Errorf("failed to create plugin dir: %w", err)
}
flags.SetDefault(config.PluginDirKey, processConfig.PluginDir)
}

flags.SetDefault(config.DataDirKey, node.DataDir)
}

// Set the network and tmpnet defaults last to ensure they can be overridden
flags.SetDefaults(n.DefaultFlags)
flags.SetDefaults(DefaultTmpnetFlags())

// Write the flags to disk
return node.writeFlags(flags)
}

// Waits until the provided nodes are healthy.
func waitForHealthy(ctx context.Context, log logging.Logger, nodes []*Node) error {
ticker := time.NewTicker(networkHealthCheckInterval)
Expand Down
2 changes: 2 additions & 0 deletions tests/fixture/tmpnet/network_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ func TestNetworkSerialization(t *testing.T) {
ctx := context.Background()

network := NewDefaultNetwork("testnet")
// Runtime configuration is required
network.DefaultRuntimeConfig.Process = &ProcessRuntimeConfig{}
// Validate round-tripping of primary subnet configuration
network.PrimarySubnetConfig = ConfigMap{
"validatorOnly": true,
Expand Down
61 changes: 55 additions & 6 deletions tests/fixture/tmpnet/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import (
"errors"
"fmt"
"io"
"maps"
"net"
"net/http"
"net/netip"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -53,12 +55,6 @@ type NodeRuntimeConfig struct {
Process *ProcessRuntimeConfig `json:"process,omitempty"`
}

type ProcessRuntimeConfig struct {
AvalancheGoPath string `json:"avalancheGoPath,omitempty"`
PluginDir string `json:"pluginDir,omitempty"`
ReuseDynamicPorts bool `json:"reuseDynamicPorts,omitempty"`
}

// Node supports configuring and running a node participating in a temporary network.
type Node struct {
// Set by EnsureNodeID which is also called when the node is read.
Expand Down Expand Up @@ -323,6 +319,59 @@ func (n *Node) GetUniqueID() string {
return n.network.UUID + "-" + strings.ToLower(nodeIDString[startIndex:endIndex])
}

// composeFlags determines the set of flags that should be used to
// start the node.
func (n *Node) composeFlags() (FlagsMap, error) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These flags are common across runtimes

flags := maps.Clone(n.Flags)

// Apply the network defaults first so that they are not overridden
flags.SetDefaults(n.network.DefaultFlags)

flags.SetDefaults(DefaultTmpnetFlags())

// Convert the network id to a string to ensure consistency in JSON round-tripping.
flags.SetDefault(config.NetworkNameKey, strconv.FormatUint(uint64(n.network.GetNetworkID()), 10))

// Set the bootstrap configuration
bootstrapIPs, bootstrapIDs := n.network.GetBootstrapIPsAndIDs(n)
flags.SetDefault(config.BootstrapIDsKey, strings.Join(bootstrapIDs, ","))
flags.SetDefault(config.BootstrapIPsKey, strings.Join(bootstrapIPs, ","))

// TODO(marun) Maybe avoid computing content flags for each node start?

if n.network.Genesis != nil {
genesisFileContent, err := n.network.GetGenesisFileContent()
if err != nil {
return nil, fmt.Errorf("failed to get genesis file content: %w", err)
}
flags.SetDefault(config.GenesisFileContentKey, genesisFileContent)

isSingleNodeNetwork := len(n.network.Nodes) == 1 && len(n.network.Genesis.InitialStakers) == 1
if isSingleNodeNetwork {
n.network.log.Info("defaulting to sybil protection disabled to enable a single-node network to start")
flags.SetDefault(config.SybilProtectionEnabledKey, "false")
}
}

subnetConfigContent, err := n.network.GetSubnetConfigContent()
if err != nil {
return nil, fmt.Errorf("failed to get subnet config content: %w", err)
}
if len(subnetConfigContent) > 0 {
flags.SetDefault(config.SubnetConfigContentKey, subnetConfigContent)
}

chainConfigContent, err := n.network.GetChainConfigContent()
if err != nil {
return nil, fmt.Errorf("failed to get chain config content: %w", err)
}
if len(chainConfigContent) > 0 {
flags.SetDefault(config.ChainConfigContentKey, chainConfigContent)
}

return flags, nil
}

// Saves the currently allocated API port to the node's configuration
// for use across restarts.
func (n *Node) SaveAPIPort() error {
Expand Down
11 changes: 0 additions & 11 deletions tests/fixture/tmpnet/node_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,6 @@ func (n *Node) GetFlagsPath() string {
return filepath.Join(n.DataDir, "flags.json")
}

func (n *Node) writeFlags(flags FlagsMap) error {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another cleanup opportunity - this method was no longer used.

bytes, err := DefaultJSONMarshal(flags)
if err != nil {
return fmt.Errorf("failed to marshal node flags: %w", err)
}
if err := os.WriteFile(n.GetFlagsPath(), bytes, perms.ReadWrite); err != nil {
return fmt.Errorf("failed to write node flags: %w", err)
}
return nil
}

func (n *Node) getConfigPath() string {
return filepath.Join(n.DataDir, defaultConfigFilename)
}
Expand Down
Loading