refactor: run order checker for bidengine in separate goroutine

due to increased number of orders (over 500M) querying via traditional way is ineffective due to - original implementation does only one query with limit set to 10000 which may put RPC into overloaded mode - this leads to provider service restart followed by teardown of all leases refs troian/pubsub#3 fixes akash-network/support#281 Signed-off-by: Artur Troian <[email protected]>
akash-network · Feb 5, 2025 · 12c35d4 · 12c35d4
1 parent 25bcb3a
commit 12c35d4
Show file tree

Hide file tree

Showing 12 changed files with 321 additions and 205 deletions.
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -71,6 +71,8 @@ jobs:
         run: |
           make modvendor
           cat "$GITHUB_WORKSPACE/env.backup" > "$GITHUB_ENV"
+      - uses: actions/setup-python@v5
+      - run: pip install tqdm
       - name: Setup direnv for provider
         run: direnv export gha >> "$GITHUB_ENV"
         working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
@@ -94,51 +96,60 @@ jobs:
           node_image: "kindest/node:${{ env.KINDEST_VERSION }}"
           cluster_name: "${{ env.KIND_NAME }}"
           config: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube/kind-config.yaml
+      - name: Setup run environment
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
+        run: |
+          direnv allow
+          cat "$GITHUB_ENV" > "$GITHUB_WORKSPACE/env.backup"
+          direnv export gha >> "$GITHUB_ENV"
       - name: Configure Kind cluster
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
-          KUSTOMIZE_INSTALLS=akash-operator-inventory make -s -C _run/kube kube-cluster-setup-e2e-ci
+          KUSTOMIZE_INSTALLS=akash-operator-inventory make kube-cluster-setup-e2e-ci
       - name: Setup K8S ingress
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
-          make -s -C _run/kube kind-k8s-ip
+          make kind-k8s-ip
       - name: K8S dump config
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           kubectl config view
       - name: K8S dump cluster info
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           kubectl cluster-info
       - name: K8S dump running pods
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           kubectl get deployments,pods,ingress,svc -A
       - name: K8S wait for operator inventory
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
-          make -s -C _run/kube kube-deployment-rollout-operator-inventory
+          make kube-deployment-rollout-operator-inventory
         env:
           KUBE_ROLLOUT_TIMEOUT: 360
+      - name: Run modvendor on akash-api
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        run: cat "$GITHUB_WORKSPACE/env.backup" > "$GITHUB_ENV"
       - name: Run E2E Tests
         working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
         run: |
           make test-e2e-integration
       - name: Print operator inventory logs
         if: always()
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           kubectl -n akash-services logs -l app.kubernetes.io/part-of=provider,app.kubernetes.io/component=operator,app.kubernetes.io/instance=inventory-service,app.kubernetes.io/name=inventory
       - name: Print operator inventory discovery logs
         if: always()
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           kubectl -n akash-services logs -l app.kubernetes.io/part-of=provider,app.kubernetes.io/component=operator,app.kubernetes.io/instance=inventory-hardware-discovery,app.kubernetes.io/name=inventory
       - name: Run K8s Tests
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           make test-k8s-integration
       - name: Post-Run K8S environment
-        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
+        working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider/_run/kube
         run: |
           kubectl get ns,pods,ingress,svc -A
diff --git a/_run/.env b/_run/.env
@@ -0,0 +1,16 @@
+AKASH_KEYRING_BACKEND=test
+AKASH_GAS_ADJUSTMENT=2
+AKASH_CHAIN_ID=local
+AKASH_YES=true
+AKASH_GAS_PRICES=0.025uakt
+AKASH_GAS=auto
+AKASH_NODE=http://localhost:26657
+
+AP_HOME=${AKASH_HOME}
+AP_KEYRING_BACKEND=${AKASH_KEYRING_BACKEND}
+AP_GAS_ADJUSTMENT=${AKASH_GAS_ADJUSTMENT}
+AP_CHAIN_ID=${AKASH_CHAIN_ID}
+AP_YES=${AKASH_YES}
+AP_GAS_PRICES=${AKASH_GAS_PRICES}
+AP_GAS=${AKASH_GAS}
+AP_NODE=${AKASH_NODE}
diff --git a/_run/.envrc b/_run/.envrc
@@ -8,5 +8,3 @@ fi
 if ! has tqdm ; then
 	echo -e "\033[31mtqdm is not installed. https://github.com/tqdm/tqdm"; exit 1
 fi
-
-export AKASH_KEYRING_BACKEND=test
diff --git a/_run/.envrc_run b/_run/.envrc_run
@@ -0,0 +1,6 @@
+AP_RUN_NAME=$(basename "$(pwd)")
+AP_RUN_DIR="${DEVCACHE_RUN}/${AP_RUN_NAME}"
+
+export AKASH_HOME="${AP_RUN_DIR}/.akash"
+
+dotenv .env
diff --git a/_run/common-base.mk b/_run/common-base.mk
@@ -1,7 +1,12 @@
 include $(abspath $(CURDIR)/../../make/init.mk)
 
-AP_RUN_NAME := $(notdir $(CURDIR))
-AP_RUN_DIR  := $(DEVCACHE_RUN)/$(AP_RUN_NAME)
+ifeq ($(AP_RUN_NAME),)
+$(error "AP_RUN_NAME is not set")
+endif
+
+ifeq ($(AP_RUN_DIR),)
+$(error "AP_RUN_DIR is not set")
+endif
 
 ifneq ($(AKASH_HOME),)
 ifneq ($(DIRENV_FILE),$(CURDIR)/.envrc)
@@ -11,7 +16,7 @@ Ensure direnv is installed and hooked to your shell profile. Refer to the docume
 ")
 endif
 else
-export AKASH_HOME = $(DEVCACHE_RUN)/$(AP_RUN_NAME)/.akash
+$(error "AKASH_HOME is not set")
 endif
 
 .PHONY: bins

diff --git a/_run/kube/.envrc b/_run/kube/.envrc
@@ -1,3 +1,4 @@
 source_up .envrc
+source_up .envrc_run
 
 export AKASH_HOME=$DEVCACHE_RUN/kube/.akash
diff --git a/bidengine/order_test.go b/bidengine/order_test.go
@@ -3,9 +3,11 @@ package bidengine
 import (
 	"context"
 	"errors"
+	"os"
 	"testing"
 	"time"
 
+	sdkclient "github.com/cosmos/cosmos-sdk/client"
 	tpubsub "github.com/troian/pubsub"
 
 	"github.com/akash-network/provider/operator/waiter"
@@ -95,10 +97,13 @@ func makeMocks(s *orderTestScaffold) {
 
 	groupResult.Group.GroupSpec.Resources[0] = resource
 
+	homeDir, _ := os.MkdirTemp("", "akash-network-test-*")
+
 	queryClientMock := &clientmocks.QueryClient{}
 	queryClientMock.On("Group", mock.Anything, mock.Anything).Return(groupResult, nil)
 	queryClientMock.On("Orders", mock.Anything, mock.Anything).Return(&mtypes.QueryOrdersResponse{}, nil)
 	queryClientMock.On("Provider", mock.Anything, mock.Anything).Return(&ptypes.QueryProviderResponse{}, nil)
+	queryClientMock.On("ClientContext").Return(sdkclient.Context{HomeDir: homeDir}, nil)
 
 	txMocks := &clientmocks.TxClient{}
 	s.broadcasts = make(chan []sdk.Msg, 1)
@@ -187,7 +192,7 @@ func makeOrderForTest(
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, fromctx.CtxKeyPubSub, tpubsub.New(ctx, 1000))
-	myService, err := NewService(ctx, mySession, scaffold.cluster, scaffold.testBus, waiter.NewNullWaiter(), cfg)
+	myService, err := NewService(ctx, scaffold.queryClient, mySession, scaffold.cluster, scaffold.testBus, waiter.NewNullWaiter(), cfg)
 	require.NoError(t, err)
 	require.NotNil(t, myService)