Skip to content

Commit 307e934

Browse files
maru-avamaru-ava
committed
[tmpnet] Enable deployment to kube
1 parent 8551e1d commit 307e934

32 files changed

+1333
-391
lines changed

‎.github/workflows/ci.yml

+31-1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,37 @@ jobs:
8484
prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
8585
loki_username: ${{ secrets.LOKI_ID || '' }}
8686
loki_password: ${{ secrets.LOKI_PASSWORD || '' }}
87+
e2e_kube:
88+
runs-on: ubuntu-latest
89+
steps:
90+
- uses: actions/checkout@v4
91+
- uses: ./.github/actions/setup-go-for-project
92+
# TODO(marun) Package this as a custom action for reuse
93+
- uses: ./.github/actions/install-nix
94+
- name: Run e2e tests
95+
shell: bash
96+
run: nix develop --command bash -x ./scripts/tests.e2e.kube.sh
97+
env:
98+
PROMETHEUS_USERNAME: ${{ secrets.PROMETHEUS_ID || '' }}
99+
PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
100+
LOKI_USERNAME: ${{ secrets.LOKI_ID || '' }}
101+
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD || '' }}
102+
GH_REPO: ${{ github.repository_owner }}/${{ github.event.repository.name }}
103+
GH_WORKFLOW: ${{ github.workflow }}
104+
GH_RUN_ID: ${{ github.run_id }}
105+
GH_RUN_NUMBER: ${{ github.run_number }}
106+
GH_RUN_ATTEMPT: ${{ github.run_attempt }}
107+
GH_JOB_ID: ${{ github.job }}
108+
- name: Export kind logs
109+
if: always()
110+
shell: bash
111+
run: kind export logs /tmp/kind-logs
112+
- name: Upload kind logs
113+
uses: actions/upload-artifact@v4
114+
with:
115+
name: kind-logs
116+
path: /tmp/kind-logs
117+
if-no-files-found: error
87118
e2e_existing_network:
88119
runs-on: ubuntu-latest
89120
steps:
@@ -250,7 +281,6 @@ jobs:
250281
- uses: actions/checkout@v4
251282
- uses: ./.github/actions/setup-go-for-project
252283
- uses: ./.github/actions/install-nix
253-
- run: nix develop --command echo "dependencies installed"
254284
- name: Run e2e tests
255285
shell: bash
256286
run: nix develop --command bash -x ./scripts/tests.e2e.bootstrap_monitor.sh

‎scripts/build_antithesis_images.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,5 @@ else
8787
"${AVALANCHE_PATH}/build/antithesis/xsvm" \
8888
"AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVAGO_PLUGIN_DIR=${AVALANCHE_PATH}/build/plugins"
8989

90-
build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
90+
build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/tests/antithesis/xsvm/Dockerfile.node"
9191
fi

‎scripts/build_xsvm_image.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
if ! [[ "$0" =~ scripts/build_xsvm_image.sh ]]; then
6+
echo "must be run from repository root"
7+
exit 255
8+
fi
9+
10+
# Directory above this script
11+
AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
12+
13+
# TODO(marun) This image name should be configurable
14+
DOCKER_IMAGE="localhost:5001/avalanchego"
15+
16+
# Build the avalancehgo node image
17+
FORCE_TAG_LATEST=1 SKIP_BUILD_RACE=1 DOCKER_IMAGE="${DOCKER_IMAGE}" ./scripts/build_image.sh
18+
19+
# TODO(marun) conditionally push the image to the registry
20+
GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
21+
docker buildx build --build-arg GO_VERSION="${GO_VERSION}" --build-arg AVALANCHEGO_NODE_IMAGE="${DOCKER_IMAGE}" \
22+
--push -t "${DOCKER_IMAGE}-xsvm" -f "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile" .

‎scripts/tests.e2e.bootstrap_monitor.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then
99
exit 255
1010
fi
1111

12+
export KUBECONFIG="$HOME/.kube/config"
13+
1214
./bin/tmpnetctl start-kind-cluster
1315

14-
KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e
16+
./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e

‎scripts/tests.e2e.kube.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
# Run e2e tests against nodes deployed to a kind cluster.
6+
7+
# TODO(marun)
8+
# - Support testing against a remote cluster
9+
10+
if ! [[ "$0" =~ scripts/tests.e2e.kube.sh ]]; then
11+
echo "must be run from repository root"
12+
exit 255
13+
fi
14+
15+
export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
16+
17+
./bin/tmpnetctl start-kind-cluster
18+
19+
if [[ -z "${SKIP_BUILD_IMAGE:-}" ]]; then
20+
bash -x ./scripts/build_xsvm_image.sh
21+
fi
22+
23+
E2E_SERIAL=1 PATH="${PWD}/bin:$PATH" bash -x ./scripts/tests.e2e.sh --runtime=kube --image-name=localhost:5001/avalanchego-xsvm

‎scripts/tests.e2e.sh

+11-8
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,17 @@ fi
2020
# the instructions to build non-portable BLST.
2121
source ./scripts/constants.sh
2222

23-
# Enable subnet testing by building xsvm
24-
./scripts/build_xsvm.sh
25-
echo ""
26-
2723
# Ensure an absolute path to avoid dependency on the working directory
2824
# of script execution.
29-
AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
30-
E2E_ARGS="--avalanchego-path=${AVALANCHEGO_PATH}"
25+
E2E_ARGS="${*:-}"
26+
if ! [[ "${E2E_ARGS}" =~ "--runtime=kube" ]]; then
27+
# If not running in kubernetes, use the local avalanchego binary
28+
AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
29+
E2E_ARGS+=" --avalanchego-path=${AVALANCHEGO_PATH}"
30+
31+
# Enable subnet testing by building the xsvm binary
32+
./scripts/build_xsvm.sh
33+
fi
3134

3235
#################################
3336
# Determine ginkgo args
@@ -58,5 +61,5 @@ else
5861
fi
5962

6063
#################################
61-
# shellcheck disable=SC2086
62-
./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- "${E2E_ARGS[@]}" "${@}"
64+
# shellcheck disable=SC2086,SC2068
65+
./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- ${E2E_ARGS[@]}

‎tests/antithesis/compose.go

+16-1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ var (
3838
// simplify usage by main entrypoints. If the provided network includes a subnet, the initial DB state for
3939
// the subnet will be created and written to the target path.
4040
func GenerateComposeConfig(network *tmpnet.Network, baseImageName string) error {
41+
// TODO(marun) Is there a better way to ensure parity between the configuration that initializes the database and the configuration used at runtime?
42+
if network.DefaultFlags == nil {
43+
network.DefaultFlags = tmpnet.FlagsMap{}
44+
}
45+
network.DefaultFlags.SetDefaults(tmpnet.DefaultTestFlags())
46+
4147
targetPath := os.Getenv(targetPathEnvName)
4248
if len(targetPath) == 0 {
4349
return errTargetPathEnvVarNotSet
@@ -66,7 +72,16 @@ func GenerateComposeConfig(network *tmpnet.Network, baseImageName string) error
6672
return fmt.Errorf("failed to get bootstrap volume path: %w", err)
6773
}
6874

69-
if err := initBootstrapDB(network, avalancheGoPath, pluginDir, bootstrapVolumePath); err != nil {
75+
network.DefaultRuntimeConfig = tmpnet.NodeRuntimeConfig{
76+
AvalancheGoPath: avalancheGoPath,
77+
}
78+
// TODO(marun) Need to have a standard way of initializing a network
79+
if network.DefaultFlags == nil {
80+
network.DefaultFlags = tmpnet.FlagsMap{}
81+
}
82+
network.DefaultFlags[config.PluginDirKey] = pluginDir
83+
84+
if err := initBootstrapDB(network, bootstrapVolumePath); err != nil {
7085
return fmt.Errorf("failed to initialize db volumes: %w", err)
7186
}
7287
}

‎tests/antithesis/init_db.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,14 @@ func getBootstrapVolumePath(targetPath string) (string, error) {
2828
// Bootstraps a local process-based network, creates its subnets and chains, and copies
2929
// the resulting db state from one of the nodes to the provided path. The path will be
3030
// created if it does not already exist.
31-
func initBootstrapDB(network *tmpnet.Network, avalancheGoPath string, pluginDir string, destPath string) error {
31+
func initBootstrapDB(network *tmpnet.Network, destPath string) error {
3232
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
3333
defer cancel()
3434
if err := tmpnet.BootstrapNewNetwork(
3535
ctx,
3636
tests.NewDefaultLogger(""),
3737
network,
3838
"",
39-
avalancheGoPath,
40-
pluginDir,
4139
); err != nil {
4240
return fmt.Errorf("failed to bootstrap network: %w", err)
4341
}

‎tests/e2e/c/dynamic_fees.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ var _ = e2e.DescribeCChain("[Dynamic Fees]", func() {
6060
env := e2e.GetEnv(tc)
6161
publicNetwork := env.GetNetwork()
6262

63-
privateNetwork := tmpnet.NewDefaultNetwork("avalanchego-e2e-dynamic-fees")
63+
privateNetwork := tmpnet.NewDefaultNetwork(tc.Log(), "avalanchego-e2e-dynamic-fees")
6464
// Copy over the defaults from the normal test suite to include settings
6565
// like the upgrade config.
6666
privateNetwork.DefaultFlags = tmpnet.FlagsMap{}

‎tests/e2e/e2e_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,5 +75,5 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
7575
// Run in every ginkgo process
7676

7777
// Initialize the local test environment from the global state
78-
e2e.InitSharedTestEnvironment(ginkgo.GinkgoT(), envBytes)
78+
e2e.InitSharedTestEnvironment(e2e.NewTestContext(), envBytes)
7979
})

‎tests/e2e/faultinjection/duplicate_node_id.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
2626
ginkgo.It("should ensure that a given Node ID (i.e. staking keypair) can be used at most once on a network", func() {
2727
network := e2e.GetEnv(tc).GetNetwork()
2828

29+
if network.DefaultRuntimeConfig.KubeRuntimeConfig != nil {
30+
// Enabling this test for kube requires supporting a flexible name mapping
31+
ginkgo.Skip("This test is not supported on kube to avoid having to deviate from composing the statefulset name with the network uuid + nodeid")
32+
}
33+
2934
tc.By("creating new node")
3035
node1 := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
3136
e2e.WaitForHealthy(tc, node1)
@@ -43,10 +48,10 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
4348
// the same node ID.
4449
config.DataDirKey: fmt.Sprintf("%s-second", node1Flags[config.DataDirKey]),
4550
}
46-
node2 := e2e.AddEphemeralNode(tc, network, node2Flags)
51+
node2 := e2e.AddEphemeralNodeWithWaitForHealth(tc, network, node2Flags, false /* waitForHealth */)
4752

4853
tc.By("checking that the second new node fails to become healthy before timeout")
49-
err := tmpnet.WaitForHealthy(tc.DefaultContext(), node2)
54+
err := tmpnet.WaitForHealthyNode(tc.DefaultContext(), tc.Log(), node2)
5055
require.ErrorIs(err, context.DeadlineExceeded)
5156

5257
tc.By("stopping the first new node")
@@ -68,7 +73,8 @@ func checkConnectedPeers(tc tests.TestContext, existingNodes []*tmpnet.Node, new
6873
require := require.New(tc)
6974

7075
// Collect the node ids of the new node's peers
71-
infoClient := info.NewClient(newNode.URI)
76+
uri := e2e.GetLocalURI(tc, newNode)
77+
infoClient := info.NewClient(uri)
7278
peers, err := infoClient.Peers(tc.DefaultContext(), nil)
7379
require.NoError(err)
7480
peerIDs := set.NewSet[ids.NodeID](len(existingNodes))
@@ -81,7 +87,8 @@ func checkConnectedPeers(tc tests.TestContext, existingNodes []*tmpnet.Node, new
8187
require.True(peerIDs.Contains(existingNode.NodeID))
8288

8389
// Check that the new node is a peer
84-
infoClient := info.NewClient(existingNode.URI)
90+
uri := e2e.GetLocalURI(tc, existingNode)
91+
infoClient := info.NewClient(uri)
8592
peers, err := infoClient.Peers(tc.DefaultContext(), nil)
8693
require.NoError(err)
8794
isPeer := false

‎tests/fixture/bootstrapmonitor/e2e/e2e_test.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -261,14 +261,16 @@ func buildImage(tc tests.TestContext, imageName string, forceNewHash bool, scrip
261261
require.NoError(err, "Image build failed: %s", output)
262262
}
263263

264-
func newNodeStatefulSet(name string, flags map[string]string) *appsv1.StatefulSet {
264+
func newNodeStatefulSet(name string, flags tmpnet.FlagsMap) *appsv1.StatefulSet {
265265
statefulSet := tmpnet.NewNodeStatefulSet(
266266
name,
267+
true, /* generateName */
267268
latestAvalanchegoImage,
268269
nodeContainerName,
269270
volumeName,
270271
volumeSize,
271272
nodeDataDir,
273+
nil,
272274
flags,
273275
)
274276

@@ -281,8 +283,9 @@ func newNodeStatefulSet(name string, flags map[string]string) *appsv1.StatefulSe
281283
return statefulSet
282284
}
283285

284-
func defaultPodFlags() map[string]string {
285-
return tmpnet.DefaultPodFlags(constants.LocalName, nodeDataDir)
286+
func defaultPodFlags() tmpnet.FlagsMap {
287+
// TODO(marun) DefaultPodFlags is only used by bootstrap monitor - maybe inline
288+
return tmpnet.DefaultPodFlags(constants.LocalName, nodeDataDir, false /* sybilProtectionEnabled */)
286289
}
287290

288291
// waitForPodCondition waits until the specified pod reports the specified condition

0 commit comments

Comments
 (0)