From ab942eeee3c22e29b2cfb1d2b14e64c6d5cdefa7 Mon Sep 17 00:00:00 2001 From: Artur Troian Date: Fri, 9 Jun 2023 08:15:52 -0400 Subject: [PATCH] feat: parse gpu attributes (#120) Signed-off-by: Artur Troian --- .github/workflows/release-dry-run.yaml | 40 --- .github/workflows/shellcheck.yaml | 18 - .github/workflows/standardize-yaml.yaml | 15 - .golangci.yaml | 3 + .../cluster_role.yaml | 64 ++++ .../akash-operator-discovery/daemonset.yaml | 46 +++ .../kustomization.yaml | 10 + .../akash-operator-discovery/rbac.yaml | 12 + .../role-binding.yaml | 17 + .../service_account.yaml | 11 + .../templates/akash-node/kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../kustomization.yaml | 2 +- .../akash-operator-ip/kustomization.yaml | 2 +- .../akash-provider/kustomization.yaml | 2 +- _run/common-kube.mk | 1 + _run/kube/Makefile | 1 + bidengine/order.go | 6 +- bidengine/order_test.go | 19 +- bidengine/pricing.go | 171 +-------- bidengine/pricing_test.go | 162 +++++++-- bidengine/shellscript.go | 188 ++++++++++ cluster/client.go | 15 +- cluster/inventory.go | 21 +- cluster/inventory_test.go | 16 +- cluster/kube/apply.go | 23 -- cluster/kube/builder/builder.go | 81 ++++- cluster/kube/builder/builder_test.go | 161 ++++++--- cluster/kube/builder/deployment.go | 51 +-- cluster/kube/builder/deployment_test.go | 15 +- cluster/kube/builder/lease_params.go | 87 +++++ cluster/kube/builder/manifest.go | 18 +- cluster/kube/builder/namespace.go | 9 +- cluster/kube/builder/netpol.go | 18 +- cluster/kube/builder/params_lease.go | 67 ---- cluster/kube/builder/podsecuritypolicy.go | 187 +++++----- cluster/kube/builder/service.go | 36 +- cluster/kube/builder/settings.go | 4 +- cluster/kube/builder/statefulset.go | 51 +-- cluster/kube/builder/workload.go | 149 ++++++-- cluster/kube/client.go | 265 +++++++------- cluster/kube/client_exec_test.go | 6 +- cluster/kube/client_test.go | 16 +- cluster/kube/deploy_test.go | 12 +- cluster/kube/invendory_node.go | 200 +++++++++++ cluster/kube/inventory.go | 333 ++++++++---------- cluster/kube/inventory_test.go | 10 +- cluster/kube/k8s_integration_test.go | 12 +- cluster/kube/resourcetypes.go | 14 +- cluster/manager.go | 113 +++--- cluster/mocks/client.go | 37 +- cluster/mocks/deployment.go | 72 +++- cluster/mocks/i_deployment.go | 165 +++++++++ cluster/mocks/reservation.go | 109 ++++++ cluster/mocks/reservation_group.go | 188 ++++++++++ cluster/monitor.go | 18 +- cluster/monitor_test.go | 65 ++-- cluster/reservation.go | 24 +- cluster/service.go | 35 +- cluster/types/v1beta3/deployment.go | 101 ++---- cluster/types/v1beta3/reservation.go | 10 +- cluster/types/v1beta3/types.go | 84 ++++- gateway/rest/integration_test.go | 10 +- gateway/rest/router.go | 15 +- gateway/rest/router_migrate_test.go | 10 +- gateway/rest/router_test.go | 114 +++--- go.mod | 23 +- go.sum | 46 +-- make/codegen.mk | 1 + make/init.mk | 12 +- make/releasing.mk | 4 +- make/test-integration.mk | 23 +- manifest/manager.go | 12 +- mocks/client.go | 33 +- pkg/apis/akash.network/crd.yaml | 100 ++---- .../v2beta2/k8s_integration_test.go | 7 +- .../v2beta2/lease_params_service.go | 45 --- pkg/apis/akash.network/v2beta2/manifest.go | 90 +++-- pkg/apis/akash.network/v2beta2/node_info.go | 24 ++ pkg/apis/akash.network/v2beta2/register.go | 4 - pkg/apis/akash.network/v2beta2/types.go | 23 +- pkg/apis/akash.network/v2beta2/types_test.go | 5 +- .../v2beta2/zz_generated.deepcopy.go | 289 +++++++-------- .../v2beta2/akash.network_client.go | 5 - .../v2beta2/fake/fake_akash.network_client.go | 4 - .../v2beta2/fake/fake_leaseparamsservice.go | 142 -------- .../v2beta2/generated_expansion.go | 2 - .../v2beta2/leaseparamsservice.go | 195 ---------- .../akash.network/v2beta2/interface.go | 7 - .../v2beta2/leaseparamsservice.go | 90 ----- .../informers/externalversions/generic.go | 2 - .../v2beta2/expansion_generated.go | 8 - .../v2beta2/leaseparamsservice.go | 99 ------ service.go | 56 ++- 94 files changed, 2945 insertions(+), 2247 deletions(-) delete mode 100644 .github/workflows/release-dry-run.yaml delete mode 100644 .github/workflows/shellcheck.yaml delete mode 100644 .github/workflows/standardize-yaml.yaml create mode 100644 _docs/kustomize/akash-operator-discovery/cluster_role.yaml create mode 100644 _docs/kustomize/akash-operator-discovery/daemonset.yaml create mode 100644 _docs/kustomize/akash-operator-discovery/kustomization.yaml create mode 100644 _docs/kustomize/akash-operator-discovery/rbac.yaml create mode 100644 _docs/kustomize/akash-operator-discovery/role-binding.yaml create mode 100644 _docs/kustomize/akash-operator-discovery/service_account.yaml create mode 100644 bidengine/shellscript.go create mode 100644 cluster/kube/builder/lease_params.go delete mode 100644 cluster/kube/builder/params_lease.go create mode 100644 cluster/kube/invendory_node.go create mode 100644 cluster/mocks/i_deployment.go create mode 100644 cluster/mocks/reservation_group.go delete mode 100644 pkg/apis/akash.network/v2beta2/lease_params_service.go create mode 100644 pkg/apis/akash.network/v2beta2/node_info.go delete mode 100644 pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_leaseparamsservice.go delete mode 100644 pkg/client/clientset/versioned/typed/akash.network/v2beta2/leaseparamsservice.go delete mode 100644 pkg/client/informers/externalversions/akash.network/v2beta2/leaseparamsservice.go delete mode 100644 pkg/client/listers/akash.network/v2beta2/leaseparamsservice.go diff --git a/.github/workflows/release-dry-run.yaml b/.github/workflows/release-dry-run.yaml deleted file mode 100644 index e3b611873..000000000 --- a/.github/workflows/release-dry-run.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: release - -defaults: - run: - shell: bash - -on: - pull_request: - push: - branches: - - main - tags: - - v* - -jobs: - dry-run: - runs-on: ubuntu-latest - env: - DOCKER_CLI_EXPERIMENTAL: "enabled" - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: c-py/action-dotenv-to-setenv@v3 - with: - env-file: .env - - uses: actions/setup-go@v3 - with: - go-version: "${{ env.GOLANG_VERSION }}" - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - uses: fregante/setup-git-user@v1 - - name: configure git tag - run: echo "RELEASE_TAG=v$(./script/semver.sh bump patch $(git describe --tags --abbrev=0))" >> $GITHUB_ENV - - name: git tag - run: git tag -a ${{ env.RELEASE_TAG }} -m ${{ env.RELEASE_TAG }} - - name: release dry-run - run: make release diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml deleted file mode 100644 index 22509cebd..000000000 --- a/.github/workflows/shellcheck.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: shellcheck - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - shellcheck-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - run: make shellcheck diff --git a/.github/workflows/standardize-yaml.yaml b/.github/workflows/standardize-yaml.yaml deleted file mode 100644 index 48c30b683..000000000 --- a/.github/workflows/standardize-yaml.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: standardize yaml -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - check-yml-files: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: check-yml-count - run: | - if [[ $(git ls-files '*.yml' ':!:.codecov.yml' | wc -l) -ne 0 ]]; then git ls-files '*.yml' ':!:.codecov.yml' && exit 1;fi diff --git a/.golangci.yaml b/.golangci.yaml index dd60ad4a3..69088480b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -8,6 +8,9 @@ run: skip-dirs: - pkg/client - ".*/mocks" + - ".*/kubernetes_mock" # Skip vendor/ etc skip-dirs-use-default: true + +linters-settings: diff --git a/_docs/kustomize/akash-operator-discovery/cluster_role.yaml b/_docs/kustomize/akash-operator-discovery/cluster_role.yaml new file mode 100644 index 000000000..fe972d199 --- /dev/null +++ b/_docs/kustomize/akash-operator-discovery/cluster_role.yaml @@ -0,0 +1,64 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: inventory-operator + labels: + akash.network: "true" + app.kubernetes.io/name: akash + app.kubernetes.io/instance: inventory + app.kubernetes.io/component: operator +rules: + - apiGroups: + - '' + resources: + - namespaces + - nodes + - pods + - events + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - pods/exec + verbs: + - create + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - ceph.rook.io + resources: + - cephclusters + - cephblockpools + verbs: + - get + - list + - watch + - apiGroups: + - akash.network + resources: + - inventoryrequests + verbs: + - get + - list + - watch + - apiGroups: + - akash.network + resources: + - inventories + verbs: + - create + - patch + - get + - list + - watch diff --git a/_docs/kustomize/akash-operator-discovery/daemonset.yaml b/_docs/kustomize/akash-operator-discovery/daemonset.yaml new file mode 100644 index 000000000..5a4a87119 --- /dev/null +++ b/_docs/kustomize/akash-operator-discovery/daemonset.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: operator-discovery + namespace: akash-services + labels: + akash.network: "true" + app.kubernetes.io/name: akash + app.kubernetes.io/instance: discovery + app.kubernetes.io/component: operator +spec: + selector: + matchLabels: + app.kubernetes.io/name: akash + app.kubernetes.io/instance: discovery + app.kubernetes.io/component: operator + replicas: 1 + revisionHistoryLimit: 1 + template: + metadata: + labels: + app: inventory-operator + app.kubernetes.io/name: akash + app.kubernetes.io/instance: discovery + app.kubernetes.io/component: operator + spec: + serviceAccountName: operator-discovery + containers: + - name: operator-discovery + image: ghcr.io/akash-network/provider + args: + - "provider-services" + - "operator" + - "inventory" + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + ports: + - containerPort: 8080 + name: api + protocol: TCP diff --git a/_docs/kustomize/akash-operator-discovery/kustomization.yaml b/_docs/kustomize/akash-operator-discovery/kustomization.yaml new file mode 100644 index 000000000..fd79eef61 --- /dev/null +++ b/_docs/kustomize/akash-operator-discovery/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: akash-services +resources: + - deployment.yaml + - service.yaml + - rbac.yaml + - service_account.yaml + - cluster_role.yaml + - role-binding.yaml diff --git a/_docs/kustomize/akash-operator-discovery/rbac.yaml b/_docs/kustomize/akash-operator-discovery/rbac.yaml new file mode 100644 index 000000000..ea1146b9a --- /dev/null +++ b/_docs/kustomize/akash-operator-discovery/rbac.yaml @@ -0,0 +1,12 @@ +#apiVersion: rbac.authorization.k8s.io/v1 +#kind: ClusterRoleBinding +#metadata: +# name: akash-ip-operator-manage-service +#subjects: +# - kind: ServiceAccount +# name: akash-ip-operator +# namespace: akash-services +#roleRef: +# kind: ClusterRole +# name: akash-ip-op-manage-service +# apiGroup: rbac.authorization.k8s.io diff --git a/_docs/kustomize/akash-operator-discovery/role-binding.yaml b/_docs/kustomize/akash-operator-discovery/role-binding.yaml new file mode 100644 index 000000000..e42e451c6 --- /dev/null +++ b/_docs/kustomize/akash-operator-discovery/role-binding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: operator-discovery + labels: + akash.network: "true" + app.kubernetes.io/name: akash + app.kubernetes.io/instance: discovery + app.kubernetes.io/component: operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: operator-discovery +subjects: + - kind: ServiceAccount + name: operator-discovery + namespace: akash-services diff --git a/_docs/kustomize/akash-operator-discovery/service_account.yaml b/_docs/kustomize/akash-operator-discovery/service_account.yaml new file mode 100644 index 000000000..f6d9befa9 --- /dev/null +++ b/_docs/kustomize/akash-operator-discovery/service_account.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator-discovery + namespace: akash-services + labels: + akash.network: "true" + app.kubernetes.io/name: akash + app.kubernetes.io/instance: discovery + app.kubernetes.io/component: operator +automountServiceAccountToken: true diff --git a/_docs/kustomize/templates/akash-node/kustomization.yaml b/_docs/kustomize/templates/akash-node/kustomization.yaml index 73b1e4e63..9d8335d7e 100644 --- a/_docs/kustomize/templates/akash-node/kustomization.yaml +++ b/_docs/kustomize/templates/akash-node/kustomization.yaml @@ -34,7 +34,7 @@ secretGenerator: - cache/config/priv_validator_key.json - cache/data/priv_validator_state.json -patchesJson6902: +patches: ## # Configure gateway host in `gateway-host.yaml`. This diff --git a/_docs/kustomize/templates/akash-operator-hostname/kustomization.yaml b/_docs/kustomize/templates/akash-operator-hostname/kustomization.yaml index b09c4de6a..b5f4f359b 100644 --- a/_docs/kustomize/templates/akash-operator-hostname/kustomization.yaml +++ b/_docs/kustomize/templates/akash-operator-hostname/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization namespace: akash-services resources: - ../../../../../_docs/kustomize/akash-operator-hostname -patchesJson6902: +patches: - path: docker-image.yaml target: kind: Deployment diff --git a/_docs/kustomize/templates/akash-operator-inventory/kustomization.yaml b/_docs/kustomize/templates/akash-operator-inventory/kustomization.yaml index df87579dc..18c55e4a8 100644 --- a/_docs/kustomize/templates/akash-operator-inventory/kustomization.yaml +++ b/_docs/kustomize/templates/akash-operator-inventory/kustomization.yaml @@ -4,7 +4,7 @@ namespace: akash-services resources: - ../../../../../_docs/kustomize/akash-operator-inventory -patchesJson6902: +patches: - path: docker-image.yaml target: kind: Deployment diff --git a/_docs/kustomize/templates/akash-operator-ip/kustomization.yaml b/_docs/kustomize/templates/akash-operator-ip/kustomization.yaml index ded0e57aa..5c872439a 100644 --- a/_docs/kustomize/templates/akash-operator-ip/kustomization.yaml +++ b/_docs/kustomize/templates/akash-operator-ip/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization namespace: akash-services resources: - ../../../../../_docs/kustomize/akash-operator-ip -patchesJson6902: +patches: - path: docker-image.yaml target: kind: Deployment diff --git a/_docs/kustomize/templates/akash-provider/kustomization.yaml b/_docs/kustomize/templates/akash-provider/kustomization.yaml index 93304009b..df21badc3 100644 --- a/_docs/kustomize/templates/akash-provider/kustomization.yaml +++ b/_docs/kustomize/templates/akash-provider/kustomization.yaml @@ -27,7 +27,7 @@ configMapGenerator: - deployment-runtime-class=none # - ingress-static-hosts=false -patchesJson6902: +patches: ## # Configure gateway host in `gateway-host.yaml`. Its value diff --git a/_run/common-kube.mk b/_run/common-kube.mk index 88b08210f..346c06302 100644 --- a/_run/common-kube.mk +++ b/_run/common-kube.mk @@ -93,6 +93,7 @@ kube-cluster-setup-e2e: $(KUBE_CREATE) kube-cluster-setup-e2e-ci .PHONY: kube-cluster-setup-e2e-ci kube-cluster-setup-e2e-ci: \ kube-setup-ingress \ + kube-prepare-images \ kube-upload-images \ kustomize-init \ kustomize-deploy-services \ diff --git a/_run/kube/Makefile b/_run/kube/Makefile index 4daf25424..153cab71c 100644 --- a/_run/kube/Makefile +++ b/_run/kube/Makefile @@ -2,6 +2,7 @@ KUBE_SETUP_PREREQUISITES ?= \ KUBE_CLUSTER_CREATE_TARGET := kind KUBE_SSH_NODE_NAME ?= kind +KUBE_DOCKER_IMAGE_ARCH := $(shell uname -m | sed "s/x86_64/amd64/g") KUSTOMIZE_INSTALLS ?= \ akash-operator-hostname \ diff --git a/bidengine/order.go b/bidengine/order.go index 93a72417a..5fb99036f 100644 --- a/bidengine/order.go +++ b/bidengine/order.go @@ -371,7 +371,11 @@ loop: } pricech = runner.Do(metricsutils.ObserveRunner(func() runner.Result { // Calculate price & bid - return runner.NewResult(o.cfg.PricingStrategy.CalculatePrice(ctx, group.GroupID.Owner, &group.GroupSpec)) + priceReq := Request{ + Owner: group.GroupID.Owner, + GSpec: &group.GroupSpec, + } + return runner.NewResult(o.cfg.PricingStrategy.CalculatePrice(ctx, priceReq)) }, pricingDuration)) case result := <-pricech: pricech = nil diff --git a/bidengine/order_test.go b/bidengine/order_test.go index 9b97dcb28..ce5f6ecfd 100644 --- a/bidengine/order_test.go +++ b/bidengine/order_test.go @@ -48,6 +48,15 @@ type orderTestScaffold struct { reserveCallNotify chan int } +type testBidPricingStrategy int64 + +type alwaysFailsBidPricingStrategy struct { + failure error +} + +var _ BidPricingStrategy = (*testBidPricingStrategy)(nil) +var _ BidPricingStrategy = (*alwaysFailsBidPricingStrategy)(nil) + func makeMocks(s *orderTestScaffold) { groupResult := &dtypes.QueryGroupResponse{} groupResult.Group.GroupSpec.Name = "testGroupName" @@ -610,9 +619,7 @@ func Test_ShouldRecognizeLeaseCreatedIfBiddingIsSkipped(t *testing.T) { require.Nil(t, broadcast) } -type testBidPricingStrategy int64 - -func (tbps testBidPricingStrategy) CalculatePrice(_ context.Context, _ string, gspec *dtypes.GroupSpec) (sdk.DecCoin, error) { +func (tbps testBidPricingStrategy) CalculatePrice(_ context.Context, _ Request) (sdk.DecCoin, error) { return sdk.NewInt64DecCoin(testutil.CoinDenom, int64(tbps)), nil } @@ -642,11 +649,7 @@ func Test_BidOrderUsesBidPricingStrategy(t *testing.T) { scaffold.cluster.AssertCalled(t, "Unreserve", scaffold.orderID, mock.Anything) } -type alwaysFailsBidPricingStrategy struct { - failure error -} - -func (afbps alwaysFailsBidPricingStrategy) CalculatePrice(_ context.Context, _ string, gspec *dtypes.GroupSpec) (sdk.DecCoin, error) { +func (afbps alwaysFailsBidPricingStrategy) CalculatePrice(_ context.Context, _ Request) (sdk.DecCoin, error) { return sdk.DecCoin{}, afbps.failure } diff --git a/bidengine/pricing.go b/bidengine/pricing.go index e556fcddf..4e6214cf0 100644 --- a/bidengine/pricing.go +++ b/bidengine/pricing.go @@ -1,15 +1,9 @@ package bidengine import ( - "bytes" "context" "crypto/rand" - "encoding/json" - "fmt" "math/big" - "os" - "os/exec" - "time" "github.com/pkg/errors" "github.com/shopspring/decimal" @@ -24,8 +18,13 @@ import ( "github.com/akash-network/provider/cluster/util" ) +type Request struct { + Owner string `json:"owner"` + GSpec *dtypes.GroupSpec +} + type BidPricingStrategy interface { - CalculatePrice(ctx context.Context, owner string, gspec *dtypes.GroupSpec) (sdk.DecCoin, error) + CalculatePrice(ctx context.Context, req Request) (sdk.DecCoin, error) } const denom = "uakt" @@ -124,9 +123,9 @@ func ceilBigRatToBigInt(v *big.Rat) *big.Int { return result } -func (fp scalePricing) CalculatePrice(_ context.Context, _ string, gspec *dtypes.GroupSpec) (sdk.DecCoin, error) { +func (fp scalePricing) CalculatePrice(_ context.Context, req Request) (sdk.DecCoin, error) { // Use unlimited precision math here. - // Otherwise a correctly crafted order could create a cost of '1' given + // Otherwise, a correctly crafted order could create a cost of '1' given // a possible configuration cpuTotal := decimal.NewFromInt(0) memoryTotal := decimal.NewFromInt(0) @@ -138,10 +137,10 @@ func (fp scalePricing) CalculatePrice(_ context.Context, _ string, gspec *dtypes endpointTotal := decimal.NewFromInt(0) ipTotal := decimal.NewFromInt(0).Add(fp.ipScale) - ipTotal = ipTotal.Mul(decimal.NewFromInt(int64(util.GetEndpointQuantityOfResourceGroup(gspec, atypes.Endpoint_LEASED_IP)))) + ipTotal = ipTotal.Mul(decimal.NewFromInt(int64(util.GetEndpointQuantityOfResourceGroup(req.GSpec, atypes.Endpoint_LEASED_IP)))) // iterate over everything & sum it up - for _, group := range gspec.Resources { + for _, group := range req.GSpec.Resources { groupCount := decimal.NewFromInt(int64(group.Count)) // Expand uint32 to int64 cpuQuantity := decimal.NewFromBigInt(group.Resources.CPU.Units.Val.BigInt(), 0) @@ -198,7 +197,7 @@ func (fp scalePricing) CalculatePrice(_ context.Context, _ string, gspec *dtypes endpointTotal = endpointTotal.Mul(fp.endpointScale) - // Each quantity must be non negative + // Each quantity must be non-negative // and fit into an Int64 if cpuTotal.IsNegative() || memoryTotal.IsNegative() || @@ -243,8 +242,8 @@ func MakeRandomRangePricing() (BidPricingStrategy, error) { return randomRangePricing(0), nil } -func (randomRangePricing) CalculatePrice(_ context.Context, _ string, gspec *dtypes.GroupSpec) (sdk.DecCoin, error) { - min, max := calculatePriceRange(gspec) +func (randomRangePricing) CalculatePrice(_ context.Context, req Request) (sdk.DecCoin, error) { + min, max := calculatePriceRange(req.GSpec) if min.IsEqual(max) { return max, nil } @@ -312,158 +311,26 @@ func calculatePriceRange(gspec *dtypes.GroupSpec) (sdk.DecCoin, sdk.DecCoin) { return sdk.NewDecCoinFromDec(rmax.Denom, cmin), sdk.NewDecCoinFromDec(rmax.Denom, cmax) } -type shellScriptPricing struct { - path string - processLimit chan int - runtimeLimit time.Duration -} - var errPathEmpty = errors.New("script path cannot be the empty string") var errProcessLimitZero = errors.New("process limit must be greater than zero") var errProcessRuntimeLimitZero = errors.New("process runtime limit must be greater than zero") -func MakeShellScriptPricing(path string, processLimit uint, runtimeLimit time.Duration) (BidPricingStrategy, error) { - if len(path) == 0 { - return nil, errPathEmpty - } - if processLimit == 0 { - return nil, errProcessLimitZero - } - if runtimeLimit == 0 { - return nil, errProcessRuntimeLimitZero - } - - result := shellScriptPricing{ - path: path, - processLimit: make(chan int, processLimit), - runtimeLimit: runtimeLimit, - } - - // Use the channel as a semaphore to limit the number of processes created for computing bid processes - // Most platforms put a limit on the number of processes a user can open. Even if the limit is high - // it isn't a good idea to open thousands of processes. - for i := uint(0); i != processLimit; i++ { - result.processLimit <- 0 - } - - return result, nil -} - type storageElement struct { Class string `json:"class"` Size uint64 `json:"size"` } +type gpuElement struct { + Units uint64 `json:"units"` + Attributes map[string]map[string]string `json:"attributes,omitempty"` +} + type dataForScriptElement struct { Memory uint64 `json:"memory"` CPU uint64 `json:"cpu"` + GPU gpuElement `json:"gpu"` Storage []storageElement `json:"storage"` Count uint32 `json:"count"` EndpointQuantity int `json:"endpoint_quantity"` IPLeaseQuantity uint `json:"ip_lease_quantity"` } - -func (ssp shellScriptPricing) CalculatePrice(ctx context.Context, owner string, gspec *dtypes.GroupSpec) (sdk.DecCoin, error) { - - buf := &bytes.Buffer{} - - dataForScript := make([]dataForScriptElement, len(gspec.Resources)) - - // iterate over everything & sum it up - for i, group := range gspec.Resources { - groupCount := group.Count - cpuQuantity := group.Resources.CPU.Units.Val.Uint64() - memoryQuantity := group.Resources.Memory.Quantity.Value() - storageQuantity := make([]storageElement, 0, len(group.Resources.Storage)) - - for _, storage := range group.Resources.Storage { - class := sdl.StorageEphemeral - if attr := storage.Attributes; attr != nil { - if cl, _ := attr.Find(sdl.StorageAttributeClass).AsString(); cl != "" { - class = cl - } - } - - storageQuantity = append(storageQuantity, storageElement{ - Class: class, - Size: storage.Quantity.Val.Uint64(), - }) - } - - endpointQuantity := len(group.Resources.Endpoints) - - dataForScript[i] = dataForScriptElement{ - CPU: cpuQuantity, - Memory: memoryQuantity, - Storage: storageQuantity, - Count: groupCount, - EndpointQuantity: endpointQuantity, - IPLeaseQuantity: util.GetEndpointQuantityOfResourceUnits(group.Resources, atypes.Endpoint_LEASED_IP), - } - } - - encoder := json.NewEncoder(buf) - err := encoder.Encode(dataForScript) - if err != nil { - return sdk.DecCoin{}, err - } - - // Take 1 from the channel - <-ssp.processLimit - defer func() { - // Always return it when this function is complete - ssp.processLimit <- 0 - }() - - processCtx, cancel := context.WithTimeout(ctx, ssp.runtimeLimit) - defer cancel() - cmd := exec.CommandContext(processCtx, ssp.path) //nolint:gosec - cmd.Stdin = buf - outputBuf := &bytes.Buffer{} - cmd.Stdout = outputBuf - stderrBuf := &bytes.Buffer{} - cmd.Stderr = stderrBuf - - subprocEnv := os.Environ() - subprocEnv = append(subprocEnv, fmt.Sprintf("AKASH_OWNER=%s", owner)) - cmd.Env = subprocEnv - - err = cmd.Run() - - if ctxErr := processCtx.Err(); ctxErr != nil { - return sdk.DecCoin{}, ctxErr - } - - if err != nil { - return sdk.DecCoin{}, fmt.Errorf("%w: script failure %s", err, stderrBuf.String()) - } - - // Decode the result - decoder := json.NewDecoder(outputBuf) - decoder.UseNumber() - - var priceNumber json.Number - err = decoder.Decode(&priceNumber) - if err != nil { - return sdk.DecCoin{}, fmt.Errorf("%w: script failure %s", err, stderrBuf.String()) - } - - price, err := sdk.NewDecFromStr(priceNumber.String()) - if err != nil { - return sdk.DecCoin{}, ErrBidQuantityInvalid - } - - if price.IsZero() { - return sdk.DecCoin{}, ErrBidZero - } - - if price.IsNegative() { - return sdk.DecCoin{}, ErrBidQuantityInvalid - } - - if !price.LTE(sdk.MaxSortableDec) { - return sdk.DecCoin{}, ErrBidQuantityInvalid - } - - return sdk.NewDecCoinFromDec(denom, price), nil -} diff --git a/bidengine/pricing_test.go b/bidengine/pricing_test.go index 3659458b6..e947c66f7 100644 --- a/bidengine/pricing_test.go +++ b/bidengine/pricing_test.go @@ -94,15 +94,16 @@ func defaultGroupSpec() *dtypes.GroupSpec { Resources: make([]dtypes.Resource, 1), } - cpu := atypes.CPU{} - cpu.Units = atypes.NewResourceValue(11) - - memory := atypes.Memory{} - memory.Quantity = atypes.NewResourceValue(10000) - clusterResources := atypes.ResourceUnits{ - CPU: &cpu, - Memory: &memory, + CPU: &atypes.CPU{ + Units: atypes.NewResourceValue(11), + }, + Memory: &atypes.Memory{ + Quantity: atypes.NewResourceValue(10000), + }, + GPU: &atypes.GPU{ + Units: atypes.NewResourceValue(0), + }, Storage: atypes.Volumes{ atypes.Storage{ Quantity: atypes.NewResourceValue(4096), @@ -130,7 +131,12 @@ func Test_ScalePricingFailsOnOverflow(t *testing.T) { require.NoError(t, err) require.NotNil(t, pricing) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + price, err := pricing.CalculatePrice(context.Background(), req) require.Equal(t, sdk.DecCoin{}, price) require.Equal(t, err, ErrBidQuantityInvalid) @@ -146,7 +152,13 @@ func Test_ScalePricingOnCpu(t *testing.T) { gspec := defaultGroupSpecCPUMem() cpuQuantity := uint64(13) gspec.Resources[0].Resources.CPU.Units = atypes.NewResourceValue(cpuQuantity) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.NotNil(t, pricing) @@ -164,7 +176,13 @@ func Test_ScalePricingOnMemory(t *testing.T) { gspec := defaultGroupSpecCPUMem() memoryQuantity := uint64(123456) gspec.Resources[0].Resources.Memory.Quantity = atypes.NewResourceValue(memoryQuantity) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) expectedPrice := testutil.AkashDecCoin(t, int64(memoryScale*memoryQuantity)) @@ -182,7 +200,11 @@ func Test_ScalePricingOnMemoryLessThanOne(t *testing.T) { // Make a resource exactly 1 byte memoryQuantity := uint64(1) gspec.Resources[0].Resources.Memory.Quantity = atypes.NewResourceValue(memoryQuantity) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) expectedPrice, err := sdk.NewDecFromStr("0.0000009536743164") @@ -192,7 +214,7 @@ func Test_ScalePricingOnMemoryLessThanOne(t *testing.T) { // Make a resource exactly 1 less byte less than two megabytes memoryQuantity = uint64(2*unit.Mi - 1) gspec.Resources[0].Resources.Memory.Quantity = atypes.NewResourceValue(memoryQuantity) - price, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + price, err = pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.NotNil(t, price) @@ -228,7 +250,11 @@ func Test_ScalePricingOnStorage(t *testing.T) { gspec := defaultGroupSpec() storageQuantity := uint64(98765) gspec.Resources[0].Resources.Storage[0].Quantity = atypes.NewResourceValue(storageQuantity) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) decNearly(t, price.Amount, int64(storageScale*storageQuantity)) @@ -247,15 +273,18 @@ func Test_ScalePricingByCountOfResources(t *testing.T) { gspec := defaultGroupSpec() storageQuantity := uint64(111) gspec.Resources[0].Resources.Storage[0].Quantity = atypes.NewResourceValue(storageQuantity) - firstPrice, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + firstPrice, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.NoError(t, err) decNearly(t, firstPrice.Amount, int64(storageScale*storageQuantity)) gspec.Resources[0].Count = 2 - - secondPrice, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + secondPrice, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) decNearly(t, secondPrice.Amount, 2*int64(storageScale*storageQuantity)) } @@ -277,7 +306,11 @@ func Test_ScalePricingForIPs(t *testing.T) { }) require.Equal(t, uint(1), util.GetEndpointQuantityOfResourceGroup(gspec, atypes.Endpoint_LEASED_IP)) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.NoError(t, err) @@ -288,14 +321,14 @@ func Test_ScalePricingForIPs(t *testing.T) { SequenceNumber: 1368, }) require.Equal(t, uint(2), util.GetEndpointQuantityOfResourceGroup(gspec, atypes.Endpoint_LEASED_IP)) - price, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + price, err = pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.NoError(t, err) decNearly(t, price.Amount, 2*ipPriceInt) gspec.Resources[0].Count = 33 // any number greater than 1 works here - price, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + price, err = pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) decNearly(t, price.Amount, 2*ipPriceInt) } @@ -328,8 +361,11 @@ func Test_ScriptPricingFailsWhenScriptDoesNotExist(t *testing.T) { pricing, err := MakeShellScriptPricing(scriptPath, 1, 30000*time.Millisecond) require.NoError(t, err) require.NotNil(t, pricing) - - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + _, err = pricing.CalculatePrice(context.Background(), req) require.IsType(t, &os.PathError{}, errors.Unwrap(err)) } @@ -344,11 +380,16 @@ func Test_ScriptPricingFailsWhenScriptExitsNonZero(t *testing.T) { err = fout.Close() require.NoError(t, err) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + pricing, err := MakeShellScriptPricing(scriptPath, 1, 30000*time.Millisecond) require.NoError(t, err) require.NotNil(t, pricing) - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + _, err = pricing.CalculatePrice(context.Background(), req) require.IsType(t, &exec.ExitError{}, errors.Unwrap(err)) } @@ -367,7 +408,12 @@ func Test_ScriptPricingFailsWhenScriptExitsWithoutWritingResultToStdout(t *testi require.NoError(t, err) require.NotNil(t, pricing) - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + _, err = pricing.CalculatePrice(context.Background(), req) require.Equal(t, io.EOF, errors.Unwrap(err)) } @@ -386,7 +432,12 @@ func Test_ScriptPricingFailsWhenScriptWritesZeroResult(t *testing.T) { require.NoError(t, err) require.NotNil(t, pricing) - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + _, err = pricing.CalculatePrice(context.Background(), req) require.Equal(t, ErrBidZero, err) } @@ -405,7 +456,12 @@ func Test_ScriptPricingFailsWhenScriptWritesNegativeResult(t *testing.T) { require.NoError(t, err) require.NotNil(t, pricing) - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + _, err = pricing.CalculatePrice(context.Background(), req) require.Equal(t, ErrBidQuantityInvalid, err) } @@ -424,7 +480,12 @@ func Test_ScriptPricingWhenScriptWritesFractionalResult(t *testing.T) { require.NoError(t, err) require.NotNil(t, pricing) - result, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + result, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) expectedPrice, err := sdk.NewDecFromStr("1.5") require.NoError(t, err) @@ -447,7 +508,12 @@ func Test_ScriptPricingFailsWhenScriptWritesOverflowResult(t *testing.T) { require.NoError(t, err) require.NotNil(t, pricing) - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + _, err = pricing.CalculatePrice(context.Background(), req) require.Equal(t, ErrBidQuantityInvalid, err) } @@ -467,7 +533,12 @@ func Test_ScriptPricingReturnsResultFromScript(t *testing.T) { require.NoError(t, err) require.NotNil(t, pricing) - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.Equal(t, "uakt", price.Denom) require.Equal(t, sdk.NewDec(132), price.Amount) @@ -491,7 +562,12 @@ func Test_ScriptPricingDoesNotExhaustSemaphore(t *testing.T) { // run the script lots of time to make sure the channel used // as a semaphore always has things returned to it for i := 0; i != 111; i++ { - _, err = pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + _, err = pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) } } @@ -513,7 +589,11 @@ func Test_ScriptPricingStopsByContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = pricing.CalculatePrice(ctx, testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + _, err = pricing.CalculatePrice(ctx, req) require.Error(t, err) require.Equal(t, context.Canceled, err) } @@ -539,7 +619,12 @@ func Test_ScriptPricingStopsByTimeout(t *testing.T) { require.NotNil(t, pricing) ctx := context.Background() - _, err = pricing.CalculatePrice(ctx, testutil.AccAddress(t).String(), defaultGroupSpec()) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: defaultGroupSpec(), + } + + _, err = pricing.CalculatePrice(ctx, req) require.Error(t, err) require.Equal(t, context.DeadlineExceeded, err) } @@ -562,7 +647,12 @@ func Test_ScriptPricingWritesJsonToStdin(t *testing.T) { require.NotNil(t, pricing) gspec := defaultGroupSpec() - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } + + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.Equal(t, "uakt", price.Denom) require.Equal(t, sdk.NewDec(1), price.Amount) @@ -614,8 +704,12 @@ func Test_ScriptPricingFromScript(t *testing.T) { gspec := defaultGroupSpec() gspec.Resources[0].Resources.Endpoints = make([]atypes.Endpoint, 7) + req := Request{ + Owner: testutil.AccAddress(t).String(), + GSpec: gspec, + } - price, err := pricing.CalculatePrice(context.Background(), testutil.AccAddress(t).String(), gspec) + price, err := pricing.CalculatePrice(context.Background(), req) require.NoError(t, err) require.Equal(t, sdk.NewDecCoin("uakt", sdk.NewInt(expectedPrice)).String(), price.String()) } diff --git a/bidengine/shellscript.go b/bidengine/shellscript.go new file mode 100644 index 000000000..42e505fe8 --- /dev/null +++ b/bidengine/shellscript.go @@ -0,0 +1,188 @@ +package bidengine + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "time" + + atypes "github.com/akash-network/akash-api/go/node/types/v1beta3" + "github.com/akash-network/node/sdl" + sdk "github.com/cosmos/cosmos-sdk/types" + + "github.com/akash-network/provider/cluster/util" +) + +type shellScriptPricing struct { + path string + processLimit chan int + runtimeLimit time.Duration +} + +func MakeShellScriptPricing(path string, processLimit uint, runtimeLimit time.Duration) (BidPricingStrategy, error) { + if len(path) == 0 { + return nil, errPathEmpty + } + if processLimit == 0 { + return nil, errProcessLimitZero + } + if runtimeLimit == 0 { + return nil, errProcessRuntimeLimitZero + } + + result := shellScriptPricing{ + path: path, + processLimit: make(chan int, processLimit), + runtimeLimit: runtimeLimit, + } + + // Use the channel as a semaphore to limit the number of processes created for computing bid processes + // Most platforms put a limit on the number of processes a user can open. Even if the limit is high + // it isn't a good idea to open thousands of processes. + for i := uint(0); i != processLimit; i++ { + result.processLimit <- 0 + } + + return result, nil +} + +func parseCPU(res *atypes.CPU) uint64 { + return res.Units.Val.Uint64() +} + +func parseMemory(res *atypes.Memory) uint64 { + return res.Quantity.Val.Uint64() +} + +func parseGPU(resource *atypes.GPU) gpuElement { + res := gpuElement{ + Units: resource.Units.Value(), + Attributes: make(map[string]map[string]string), + } + + for _, attr := range resource.Attributes { + tokens := strings.Split(attr.Key, "/") + if _, exists := res.Attributes[tokens[0]]; !exists { + res.Attributes[tokens[0]] = make(map[string]string) + } + res.Attributes[tokens[0]][tokens[1]] = tokens[2] + } + + return res +} + +func parseStorage(resource atypes.Volumes) []storageElement { + res := make([]storageElement, 0, len(resource)) + + for _, storage := range resource { + class := sdl.StorageEphemeral + if attr := storage.Attributes; attr != nil { + if cl, _ := attr.Find(sdl.StorageAttributeClass).AsString(); cl != "" { + class = cl + } + } + + res = append(res, storageElement{ + Class: class, + Size: storage.Quantity.Val.Uint64(), + }) + } + + return res +} + +func (ssp shellScriptPricing) CalculatePrice(ctx context.Context, req Request) (sdk.DecCoin, error) { + buf := &bytes.Buffer{} + + dataForScript := make([]dataForScriptElement, len(req.GSpec.Resources)) + + // iterate over everything & sum it up + for i, group := range req.GSpec.Resources { + groupCount := group.Count + + cpuQuantity := parseCPU(group.Resources.CPU) + gpuQuantity := parseGPU(group.Resources.GPU) + memoryQuantity := parseMemory(group.Resources.Memory) + storageQuantity := parseStorage(group.Resources.Storage) + endpointQuantity := len(group.Resources.Endpoints) + + dataForScript[i] = dataForScriptElement{ + CPU: cpuQuantity, + GPU: gpuQuantity, + Memory: memoryQuantity, + Storage: storageQuantity, + Count: groupCount, + EndpointQuantity: endpointQuantity, + IPLeaseQuantity: util.GetEndpointQuantityOfResourceUnits(group.Resources, atypes.Endpoint_LEASED_IP), + } + } + + encoder := json.NewEncoder(buf) + err := encoder.Encode(dataForScript) + if err != nil { + return sdk.DecCoin{}, err + } + + // Take 1 from the channel + <-ssp.processLimit + defer func() { + // Always return it when this function is complete + ssp.processLimit <- 0 + }() + + processCtx, cancel := context.WithTimeout(ctx, ssp.runtimeLimit) + defer cancel() + cmd := exec.CommandContext(processCtx, ssp.path) //nolint:gosec + cmd.Stdin = buf + outputBuf := &bytes.Buffer{} + cmd.Stdout = outputBuf + stderrBuf := &bytes.Buffer{} + cmd.Stderr = stderrBuf + + subprocEnv := os.Environ() + subprocEnv = append(subprocEnv, fmt.Sprintf("AKASH_OWNER=%s", req.Owner)) + cmd.Env = subprocEnv + + err = cmd.Run() + + if ctxErr := processCtx.Err(); ctxErr != nil { + return sdk.DecCoin{}, ctxErr + } + + if err != nil { + return sdk.DecCoin{}, fmt.Errorf("%w: script failure %s", err, stderrBuf.String()) + } + + // Decode the result + decoder := json.NewDecoder(outputBuf) + decoder.UseNumber() + + var priceNumber json.Number + err = decoder.Decode(&priceNumber) + if err != nil { + return sdk.DecCoin{}, fmt.Errorf("%w: script failure %s", err, stderrBuf.String()) + } + + price, err := sdk.NewDecFromStr(priceNumber.String()) + if err != nil { + return sdk.DecCoin{}, ErrBidQuantityInvalid + } + + if price.IsZero() { + return sdk.DecCoin{}, ErrBidZero + } + + if price.IsNegative() { + return sdk.DecCoin{}, ErrBidQuantityInvalid + } + + if !price.LTE(sdk.MaxSortableDec) { + return sdk.DecCoin{}, ErrBidQuantityInvalid + } + + return sdk.NewDecCoinFromDec(denom, price), nil +} diff --git a/cluster/client.go b/cluster/client.go index 9fd075b09..da72972aa 100644 --- a/cluster/client.go +++ b/cluster/client.go @@ -66,9 +66,9 @@ type ReadClient interface { //go:generate mockery --name Client type Client interface { ReadClient - Deploy(ctx context.Context, lID mtypes.LeaseID, mgroup *mani.Group) error + Deploy(ctx context.Context, deployment ctypes.IDeployment) error TeardownLease(context.Context, mtypes.LeaseID) error - Deployments(context.Context) ([]ctypes.Deployment, error) + Deployments(context.Context) ([]ctypes.IDeployment, error) Inventory(context.Context) (ctypes.Inventory, error) Exec(ctx context.Context, lID mtypes.LeaseID, @@ -171,7 +171,7 @@ type inventory struct { var _ ctypes.Inventory = (*inventory)(nil) -func (inv *inventory) Adjust(reservation ctypes.Reservation) error { +func (inv *inventory) Adjust(reservation ctypes.ReservationGroup, opts ...ctypes.InventoryOption) error { resources := make([]types.Resources, len(reservation.Resources().GetResources())) copy(resources, reservation.Resources().GetResources()) @@ -407,7 +407,7 @@ func (c *nullClient) RemoveHostnameFromDeployment(_ context.Context, _ string, _ func (c *nullClient) ObserveHostnameState(_ context.Context) (<-chan ctypes.HostnameResourceEvent, error) { return nil, errNotImplemented } -func (c *nullClient) GetDeployments(_ context.Context, _ dtypes.DeploymentID) ([]ctypes.Deployment, error) { +func (c *nullClient) GetDeployments(_ context.Context, _ dtypes.DeploymentID) ([]ctypes.IDeployment, error) { return nil, errNotImplemented } @@ -431,10 +431,13 @@ func (c *nullClient) PurgeDeclaredHostname(_ context.Context, _ mtypes.LeaseID, return errNotImplemented } -func (c *nullClient) Deploy(ctx context.Context, lid mtypes.LeaseID, mgroup *mani.Group) error { +func (c *nullClient) Deploy(ctx context.Context, deployment ctypes.IDeployment) error { c.mtx.Lock() defer c.mtx.Unlock() + lid := deployment.LeaseID() + mgroup := deployment.ManifestGroup() + ctx, cancel := context.WithCancel(ctx) c.leases[mquery.LeasePath(lid)] = &nullLease{ ctx: ctx, @@ -551,7 +554,7 @@ func (c *nullClient) TeardownLease(_ context.Context, lid mtypes.LeaseID) error return nil } -func (c *nullClient) Deployments(context.Context) ([]ctypes.Deployment, error) { +func (c *nullClient) Deployments(context.Context) ([]ctypes.IDeployment, error) { return nil, nil } diff --git a/cluster/inventory.go b/cluster/inventory.go index 31b538bed..d5a8916a5 100644 --- a/cluster/inventory.go +++ b/cluster/inventory.go @@ -1,7 +1,6 @@ package cluster import ( - "bytes" "context" "encoding/json" "errors" @@ -94,7 +93,7 @@ func newInventoryService( client Client, ipOperatorClient operatorclients.IPOperatorClient, waiter waiter.OperatorWaiter, - deployments []ctypes.Deployment, + deployments []ctypes.IDeployment, ) (*inventoryService, error) { sub, err := sub.Clone() @@ -161,9 +160,9 @@ func (is *inventoryService) reserve(order mtypes.OrderID, resources atypes.Resou if res.Resources.CPU == nil { return nil, fmt.Errorf("%w: CPU resource at idx %d is nil", ErrInvalidResource, idx) } - // if res.Resources.GPU == nil { - // return nil, fmt.Errorf("%w: GPU resource at idx %d is nil", ErrInvalidResource, idx) - // } + if res.Resources.GPU == nil { + return nil, fmt.Errorf("%w: GPU resource at idx %d is nil", ErrInvalidResource, idx) + } if res.Resources.Memory == nil { return nil, fmt.Errorf("%w: Memory resource at idx %d is nil", ErrInvalidResource, idx) } @@ -389,7 +388,10 @@ func (is *inventoryService) handleRequest(req inventoryRequest, state *inventory // create new registration if capacity available reservation := newReservation(req.order, resourcesToCommit) - is.log.Debug("reservation requested", "order", req.order, "resources", req.resources) + { + jReservation, _ := json.Marshal(req.resources.GetResources()) + is.log.Debug("reservation requested", "order", req.order, "resources", jReservation) + } if reservation.endpointQuantity != 0 { if is.ipOperator == nil { @@ -593,11 +595,9 @@ loop: is.updateInventoryMetrics(metrics) if fetchCount%is.config.InventoryResourceDebugFrequency == 0 { - buf := &bytes.Buffer{} - enc := json.NewEncoder(buf) - err := enc.Encode(&metrics) + data, err := json.Marshal(&metrics) if err == nil { - is.log.Debug("cluster resources", "dump", buf.String()) + is.log.Debug("cluster resources", "dump", data) } else { is.log.Error("unable to dump cluster inventory", "error", err.Error()) } @@ -607,6 +607,7 @@ loop: // readjust inventory accordingly with pending leases for _, r := range state.reservations { if !r.allocated { + // FIXME check if call for Adjust actually needed to be here if err := state.inventory.Adjust(r); err != nil { is.log.Error("adjust inventory for pending reservation", "error", err.Error()) } diff --git a/cluster/inventory_test.go b/cluster/inventory_test.go index 0dbdfbad6..41c3495cf 100644 --- a/cluster/inventory_test.go +++ b/cluster/inventory_test.go @@ -129,7 +129,7 @@ func TestInventory_ClusterDeploymentNotDeployed(t *testing.T) { subscriber, err := bus.Subscribe() require.NoError(t, err) - deployments := make([]ctypes.Deployment, 0) + deployments := make([]ctypes.IDeployment, 0) clusterClient := &mocks.Client{} @@ -169,8 +169,8 @@ func TestInventory_ClusterDeploymentDeployed(t *testing.T) { subscriber, err := bus.Subscribe() require.NoError(t, err) - deployments := make([]ctypes.Deployment, 1) - deployment := &mocks.Deployment{} + deployments := make([]ctypes.IDeployment, 1) + deployment := &mocks.IDeployment{} deployment.On("LeaseID").Return(lid) groupServices := make(manifest.Services, 1) @@ -213,7 +213,7 @@ func TestInventory_ClusterDeploymentDeployed(t *testing.T) { Services: groupServices, } - deployment.On("ManifestGroup").Return(group) + deployment.On("ManifestGroup").Return(&group) deployments[0] = deployment clusterClient := &mocks.Client{} @@ -449,7 +449,7 @@ func TestInventory_ReserveIPNoIPOperator(t *testing.T) { scaffold.clusterClient, nil, // No IP operator client waiter.NewNullWaiter(), // Do not need to wait in test - make([]ctypes.Deployment, 0)) + make([]ctypes.IDeployment, 0)) require.NoError(t, err) require.NotNil(t, inv) @@ -494,7 +494,7 @@ func TestInventory_ReserveIPUnavailableWithIPOperator(t *testing.T) { scaffold.clusterClient, mockIP, waiter.NewNullWaiter(), // Do not need to wait in test - make([]ctypes.Deployment, 0)) + make([]ctypes.IDeployment, 0)) require.NoError(t, err) require.NotNil(t, inv) @@ -557,7 +557,7 @@ func TestInventory_ReserveIPAvailableWithIPOperator(t *testing.T) { scaffold.clusterClient, mockIP, waiter.NewNullWaiter(), // Do not need to wait in test - make([]ctypes.Deployment, 0)) + make([]ctypes.IDeployment, 0)) require.NoError(t, err) require.NotNil(t, inv) @@ -661,7 +661,7 @@ func TestInventory_OverReservations(t *testing.T) { scaffold.clusterClient, nil, // No IP operator client waiter.NewNullWaiter(), // Do not need to wait in test - make([]ctypes.Deployment, 0)) + make([]ctypes.IDeployment, 0)) require.NoError(t, err) require.NotNil(t, inv) diff --git a/cluster/kube/apply.go b/cluster/kube/apply.go index aa63b35d5..6d7167a1c 100644 --- a/cluster/kube/apply.go +++ b/cluster/kube/apply.go @@ -175,26 +175,3 @@ func applyManifest(ctx context.Context, kc crdapi.Interface, b builder.Manifest) } return err } - -func applyServicesParams(ctx context.Context, kc crdapi.Interface, b builder.ParamsServices) error { - obj, err := kc.AkashV2beta2().LeaseParamsServices(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{}) - - metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "akash-parameters-get", err, errors.IsNotFound) - - switch { - case err == nil: - // TODO - only run this update if it would change something - obj, err = b.Update(obj) - if err == nil { - _, err = kc.AkashV2beta2().LeaseParamsServices(b.NS()).Update(ctx, obj, metav1.UpdateOptions{}) - metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "akash-parameters-update", err) - } - case errors.IsNotFound(err): - obj, err = b.Create() - if err == nil { - _, err = kc.AkashV2beta2().LeaseParamsServices(b.NS()).Create(ctx, obj, metav1.CreateOptions{}) - metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "akash-parameters-create", err) - } - } - return err -} diff --git a/cluster/kube/builder/builder.go b/cluster/kube/builder/builder.go index bc31621f2..e460e935d 100644 --- a/cluster/kube/builder/builder.go +++ b/cluster/kube/builder/builder.go @@ -2,16 +2,18 @@ package builder import ( "fmt" + "reflect" "strconv" - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" - "github.com/tendermint/tendermint/libs/log" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" + "github.com/tendermint/tendermint/libs/log" + + mani "github.com/akash-network/akash-api/go/manifest/v2beta2" mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" clusterUtil "github.com/akash-network/provider/cluster/util" crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) @@ -21,16 +23,16 @@ const ( AkashManifestServiceLabelName = "akash.network/manifest-service" AkashNetworkStorageClasses = "akash.network/storageclasses" AkashServiceTarget = "akash.network/service-target" + AkashServiceCapabilityGPU = "akash.network/capability/gpu" AkashMetalLB = "metal-lb" akashDeploymentPolicyName = "akash-deployment-restrictions" - - akashNetworkNamespace = "akash.network/namespace" - AkashLeaseOwnerLabelName = "akash.network/lease.id.owner" - AkashLeaseDSeqLabelName = "akash.network/lease.id.dseq" - AkashLeaseGSeqLabelName = "akash.network/lease.id.gseq" - AkashLeaseOSeqLabelName = "akash.network/lease.id.oseq" - AkashLeaseProviderLabelName = "akash.network/lease.id.provider" - AkashLeaseManifestVersion = "akash.network/manifest.version" + akashNetworkNamespace = "akash.network/namespace" + AkashLeaseOwnerLabelName = "akash.network/lease.id.owner" + AkashLeaseDSeqLabelName = "akash.network/lease.id.dseq" + AkashLeaseGSeqLabelName = "akash.network/lease.id.gseq" + AkashLeaseOSeqLabelName = "akash.network/lease.id.oseq" + AkashLeaseProviderLabelName = "akash.network/lease.id.provider" + AkashLeaseManifestVersion = "akash.network/manifest.version" ) const ( @@ -53,6 +55,51 @@ var ( tcpProtocol = corev1.Protocol("TCP") ) +type IClusterDeployment interface { + LeaseID() mtypes.LeaseID + ManifestGroup() *mani.Group + ClusterParams() crd.ClusterSettings +} + +type ClusterDeployment struct { + Lid mtypes.LeaseID + Group *mani.Group + Sparams crd.ClusterSettings +} + +var _ IClusterDeployment = (*ClusterDeployment)(nil) + +func ClusterDeploymentFromDeployment(d ctypes.IDeployment) (IClusterDeployment, error) { + cparams, valid := d.ClusterParams().(crd.ClusterSettings) + if !valid { + // nolint: goerr113 + return nil, fmt.Errorf("kube-cluster: unexpected type from ClusterParams(). expected (%s), actual (%s)", + reflect.TypeOf(crd.ClusterSettings{}), + reflect.TypeOf(d.ClusterParams()), + ) + } + + cd := &ClusterDeployment{ + Lid: d.LeaseID(), + Group: d.ManifestGroup(), + Sparams: cparams, + } + + return cd, nil +} + +func (d *ClusterDeployment) LeaseID() mtypes.LeaseID { + return d.Lid +} + +func (d *ClusterDeployment) ManifestGroup() *mani.Group { + return d.Group +} + +func (d *ClusterDeployment) ClusterParams() crd.ClusterSettings { + return d.Sparams +} + type builderBase interface { NS() string Name() string @@ -60,17 +107,15 @@ type builderBase interface { } type builder struct { - log log.Logger - settings Settings - lid mtypes.LeaseID - group *mani.Group - sparams crd.ParamsServices + log log.Logger + settings Settings + deployment IClusterDeployment } var _ builderBase = (*builder)(nil) func (b *builder) NS() string { - return LidNS(b.lid) + return LidNS(b.deployment.LeaseID()) } func (b *builder) Name() string { @@ -80,7 +125,7 @@ func (b *builder) Name() string { func (b *builder) labels() map[string]string { return map[string]string{ AkashManagedLabelName: "true", - akashNetworkNamespace: LidNS(b.lid), + akashNetworkNamespace: LidNS(b.deployment.LeaseID()), } } diff --git a/cluster/kube/builder/builder_test.go b/cluster/kube/builder/builder_test.go index 3f900bc90..d24755f52 100644 --- a/cluster/kube/builder/builder_test.go +++ b/cluster/kube/builder/builder_test.go @@ -10,25 +10,32 @@ import ( manitypes "github.com/akash-network/akash-api/go/manifest/v2beta2" "github.com/akash-network/node/testutil" + + "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) const testKubeClientNs = "nstest1111" func TestLidNsSanity(t *testing.T) { log := testutil.Logger(t) - leaseID := testutil.LeaseID(t) - settings := NewDefaultSettings() - g := &manitypes.Group{} - mb := BuildManifest(log, settings, testKubeClientNs, leaseID, g) + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{}, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{}, + }, + } + + mb := BuildManifest(log, settings, testKubeClientNs, cdep) require.Equal(t, testKubeClientNs, mb.NS()) m, err := mb.Create() require.NoError(t, err) - require.Equal(t, m.Spec.LeaseID.DSeq, strconv.FormatUint(leaseID.DSeq, 10)) + require.Equal(t, m.Spec.LeaseID.DSeq, strconv.FormatUint(cdep.Lid.DSeq, 10)) - require.Equal(t, LidNS(leaseID), m.Name) + require.Equal(t, LidNS(cdep.Lid), m.Name) } // func TestNetworkPolicies(t *testing.T) { @@ -65,17 +72,26 @@ func TestLidNsSanity(t *testing.T) { func TestGlobalServiceBuilder(t *testing.T) { myLog := testutil.Logger(t) - group := &manitypes.Group{ - Services: manitypes.Services{ - manitypes.Service{ - Name: "myservice", + mySettings := NewDefaultSettings() + + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{ + Services: manitypes.Services{ + manitypes.Service{ + Name: "myservice", + }, + }, + }, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{ + nil, }, }, } - mySettings := NewDefaultSettings() - lid := testutil.LeaseID(t) - serviceBuilder := BuildService(myLog, mySettings, lid, group, 0, true) + workload := NewWorkloadBuilder(myLog, mySettings, cdep, 0) + serviceBuilder := BuildService(workload, true) require.NotNil(t, serviceBuilder) // Should have name ending with suffix require.Equal(t, "myservice-np", serviceBuilder.Name()) @@ -85,16 +101,24 @@ func TestGlobalServiceBuilder(t *testing.T) { func TestLocalServiceBuilder(t *testing.T) { myLog := testutil.Logger(t) - group := &manitypes.Group{ - Services: manitypes.Services{ - manitypes.Service{ - Name: "myservice", + mySettings := NewDefaultSettings() + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{ + Services: manitypes.Services{ + manitypes.Service{ + Name: "myservice", + }, + }, + }, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{ + nil, }, }, } - mySettings := NewDefaultSettings() - lid := testutil.LeaseID(t) - serviceBuilder := BuildService(myLog, mySettings, lid, group, 0, false) + + serviceBuilder := BuildService(NewWorkloadBuilder(myLog, mySettings, cdep, 0), false) require.NotNil(t, serviceBuilder) // Should have name verbatim require.Equal(t, "myservice", serviceBuilder.Name()) @@ -106,18 +130,27 @@ func TestGlobalServiceBuilderWithoutGlobalServices(t *testing.T) { myLog := testutil.Logger(t) exposesServices := make([]manitypes.ServiceExpose, 1) exposesServices[0].Global = false - group := &manitypes.Group{ - Services: manitypes.Services{ - manitypes.Service{ - Name: "myservice", - Expose: exposesServices, + + mySettings := NewDefaultSettings() + + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{ + Services: manitypes.Services{ + manitypes.Service{ + Name: "myservice", + Expose: exposesServices, + }, + }, + }, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{ + nil, }, }, } - mySettings := NewDefaultSettings() - lid := testutil.LeaseID(t) - serviceBuilder := BuildService(myLog, mySettings, lid, group, 0, true) + serviceBuilder := BuildService(NewWorkloadBuilder(myLog, mySettings, cdep, 0), true) // Should not have any work to do require.False(t, serviceBuilder.Any()) @@ -138,18 +171,27 @@ func TestGlobalServiceBuilderWithGlobalServices(t *testing.T) { Port: 2000, ExternalPort: 2001, } - group := &manitypes.Group{ - Services: manitypes.Services{ - manitypes.Service{ - Name: "myservice", - Expose: exposesServices, + + mySettings := NewDefaultSettings() + + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{ + Services: manitypes.Services{ + manitypes.Service{ + Name: "myservice", + Expose: exposesServices, + }, + }, + }, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{ + nil, }, }, } - mySettings := NewDefaultSettings() - lid := testutil.LeaseID(t) - serviceBuilder := BuildService(myLog, mySettings, lid, group, 0, true) + serviceBuilder := BuildService(NewWorkloadBuilder(myLog, mySettings, cdep, 0), true) // Should have work to do require.True(t, serviceBuilder.Any()) @@ -169,18 +211,25 @@ func TestLocalServiceBuilderWithoutLocalServices(t *testing.T) { exposesServices := make([]manitypes.ServiceExpose, 1) exposesServices[0].Global = true - group := &manitypes.Group{ - Services: manitypes.Services{ - manitypes.Service{ - Name: "myservice", - Expose: exposesServices, + mySettings := NewDefaultSettings() + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{ + Services: manitypes.Services{ + manitypes.Service{ + Name: "myservice", + Expose: exposesServices, + }, + }, + }, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{ + nil, }, }, } - mySettings := NewDefaultSettings() - lid := testutil.LeaseID(t) - serviceBuilder := BuildService(myLog, mySettings, lid, group, 0, false) + serviceBuilder := BuildService(NewWorkloadBuilder(myLog, mySettings, cdep, 0), false) // Should have work to do require.False(t, serviceBuilder.Any()) @@ -201,18 +250,26 @@ func TestLocalServiceBuilderWithLocalServices(t *testing.T) { Port: 2000, ExternalPort: 2001, } - group := &manitypes.Group{ - Services: manitypes.Services{ - manitypes.Service{ - Name: "myservice", - Expose: exposesServices, + + mySettings := NewDefaultSettings() + cdep := &ClusterDeployment{ + Lid: testutil.LeaseID(t), + Group: &manitypes.Group{ + Services: manitypes.Services{ + manitypes.Service{ + Name: "myservice", + Expose: exposesServices, + }, + }, + }, + Sparams: v2beta2.ClusterSettings{ + SchedulerParams: []*v2beta2.SchedulerParams{ + nil, }, }, } - mySettings := NewDefaultSettings() - lid := testutil.LeaseID(t) - serviceBuilder := BuildService(myLog, mySettings, lid, group, 0, false) + serviceBuilder := BuildService(NewWorkloadBuilder(myLog, mySettings, cdep, 0), false) // Should have work to do require.True(t, serviceBuilder.Any()) diff --git a/cluster/kube/builder/deployment.go b/cluster/kube/builder/deployment.go index 0708fc33f..f337a36b1 100644 --- a/cluster/kube/builder/deployment.go +++ b/cluster/kube/builder/deployment.go @@ -1,15 +1,9 @@ package builder import ( - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" - "github.com/tendermint/tendermint/libs/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" - - crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) type Deployment interface { @@ -19,39 +13,24 @@ type Deployment interface { } type deployment struct { - workload + Workload } var _ Deployment = (*deployment)(nil) -func NewDeployment( - log log.Logger, - settings Settings, - lid mtypes.LeaseID, - group *mani.Group, - sparams crd.ParamsServices, - serviceIdx int) Deployment { - return &deployment{ - workload: newWorkloadBuilder(log, settings, lid, group, sparams, serviceIdx), +func NewDeployment(workload Workload) Deployment { + ss := &deployment{ + Workload: workload, } + + ss.Workload.log = ss.Workload.log.With("object", "deployment", "service-name", ss.deployment.ManifestGroup().Services[ss.serviceIdx].Name) + + return ss } func (b *deployment) Create() (*appsv1.Deployment, error) { // nolint:golint,unparam - service := &b.group.Services[b.serviceIdx] - params := &b.sparams[b.serviceIdx].Params - - replicas := int32(service.Count) falseValue := false - // fixme b.runtimeClassName is updated on call to the container() - containers := []corev1.Container{b.container()} - - var effectiveRuntimeClassName *string - if len(params.RuntimeClass) != 0 && params.RuntimeClass != runtimeClassNoneValue { - runtimeClass := params.RuntimeClass - effectiveRuntimeClassName = &runtimeClass - } - kdeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: b.Name(), @@ -61,18 +40,19 @@ func (b *deployment) Create() (*appsv1.Deployment, error) { // nolint:golint,unp Selector: &metav1.LabelSelector{ MatchLabels: b.labels(), }, - Replicas: &replicas, + Replicas: b.replicas(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: b.labels(), }, Spec: corev1.PodSpec{ - RuntimeClassName: effectiveRuntimeClassName, + Affinity: b.affinity(), + RuntimeClassName: b.runtimeClass(), SecurityContext: &corev1.PodSecurityContext{ RunAsNonRoot: &falseValue, }, AutomountServiceAccountToken: &falseValue, - Containers: containers, + Containers: []corev1.Container{b.container()}, ImagePullSecrets: b.imagePullSecrets(), }, }, @@ -83,13 +63,12 @@ func (b *deployment) Create() (*appsv1.Deployment, error) { // nolint:golint,unp } func (b *deployment) Update(obj *appsv1.Deployment) (*appsv1.Deployment, error) { // nolint:golint,unparam - service := &b.group.Services[b.serviceIdx] - replicas := int32(service.Count) - obj.Labels = b.labels() obj.Spec.Selector.MatchLabels = b.labels() - obj.Spec.Replicas = &replicas + obj.Spec.Replicas = b.replicas() obj.Spec.Template.Labels = b.labels() + obj.Spec.Template.Spec.Affinity = b.affinity() + obj.Spec.Template.Spec.RuntimeClassName = b.runtimeClass() obj.Spec.Template.Spec.Containers = []corev1.Container{b.container()} obj.Spec.Template.Spec.ImagePullSecrets = b.imagePullSecrets() diff --git a/cluster/kube/builder/deployment_test.go b/cluster/kube/builder/deployment_test.go index 941dff1a5..ee7985274 100644 --- a/cluster/kube/builder/deployment_test.go +++ b/cluster/kube/builder/deployment_test.go @@ -4,11 +4,11 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/akash-network/node/sdl" "github.com/akash-network/node/testutil" - "github.com/stretchr/testify/require" - crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) @@ -25,9 +25,16 @@ func TestDeploySetsEnvironmentVariables(t *testing.T) { mani, err := sdl.Manifest() require.NoError(t, err) - sparams := make(crd.ParamsServices, len(mani.GetGroups()[0].Services)) + sparams := make([]*crd.SchedulerParams, len(mani.GetGroups()[0].Services)) + group := mani.GetGroups()[0] + + cdep := &ClusterDeployment{ + Lid: lid, + Group: &group, + Sparams: crd.ClusterSettings{SchedulerParams: sparams}, + } - deploymentBuilder := NewDeployment(log, settings, lid, &mani.GetGroups()[0], sparams, 0) + deploymentBuilder := NewDeployment(NewWorkloadBuilder(log, settings, cdep, 0)) require.NotNil(t, deploymentBuilder) diff --git a/cluster/kube/builder/lease_params.go b/cluster/kube/builder/lease_params.go new file mode 100644 index 000000000..da7eb396b --- /dev/null +++ b/cluster/kube/builder/lease_params.go @@ -0,0 +1,87 @@ +package builder + +// type ParamsServices interface { +// builderBase +// Create() (*crd.LeaseParamsService, error) +// Update(obj *crd.LeaseParamsService) (*crd.LeaseParamsService, error) +// Params() crd.ParamsServices +// Name() string +// } +// +// type paramsServices struct { +// builder +// ns string +// } +// +// var _ ParamsServices = (*paramsServices)(nil) +// +// func BuildParamsLeases(log log.Logger, settings Settings, ns string, lid mtypes.LeaseID, group *mapi.Group) ParamsServices { +// params := make(crd.ParamsServices, 0, len(group.Services)) +// for _, svc := range group.Services { +// sParams := crd.ParamsService{ +// Name: svc.Name, +// } +// +// gpu := svc.Resources.GPU +// if gpu.Units.Value() > 0 { +// attrs, _ := ctypes.ParseGPUAttributes(gpu.Attributes) +// runtimeClass := "" +// +// if _, exists := attrs["nvidia"]; exists { +// runtimeClass = runtimeClassNvidia +// } +// +// sParams.Params = &crd.Params{ +// RuntimeClass: runtimeClass, +// Affinity: nil, +// } +// } +// +// params = append(params, sParams) +// } +// +// return ¶msServices{ +// builder: builder{ +// log: log.With("module", "kube-builder"), +// settings: settings, +// lid: lid, +// sparams: params, +// }, +// ns: ns, +// } +// } +// +// func (b *paramsServices) Create() (*crd.LeaseParamsService, error) { +// obj := &crd.LeaseParamsService{ +// ObjectMeta: metav1.ObjectMeta{ +// Namespace: b.ns, +// Name: b.Name(), +// Labels: b.labels(), +// }, +// Spec: crd.LeaseParamsServiceSpec{ +// LeaseID: crd.LeaseIDFromAkash(b.lid), +// Services: b.sparams, +// }, +// } +// +// return obj, nil +// } +// +// func (b *paramsServices) Update(obj *crd.LeaseParamsService) (*crd.LeaseParamsService, error) { +// cobj := *obj +// cobj.Spec.Services = b.sparams +// +// return &cobj, nil +// } +// +// func (b *paramsServices) NS() string { +// return b.ns +// } +// +// func (b *paramsServices) Params() crd.ParamsServices { +// return b.sparams +// } +// +// func (b *paramsServices) labels() map[string]string { +// return AppendLeaseLabels(b.lid, b.builder.labels()) +// } diff --git a/cluster/kube/builder/manifest.go b/cluster/kube/builder/manifest.go index 070a064e6..5b9ef5119 100644 --- a/cluster/kube/builder/manifest.go +++ b/cluster/kube/builder/manifest.go @@ -1,11 +1,8 @@ package builder import ( - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" "github.com/tendermint/tendermint/libs/log" - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" - crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) @@ -25,24 +22,23 @@ type manifest struct { var _ Manifest = (*manifest)(nil) -func BuildManifest(log log.Logger, settings Settings, ns string, lid mtypes.LeaseID, group *mani.Group) Manifest { +func BuildManifest(log log.Logger, settings Settings, ns string, deployment IClusterDeployment) Manifest { return &manifest{ builder: builder{ - log: log.With("module", "kube-builder"), - settings: settings, - lid: lid, - group: group, + log: log.With("module", "kube-builder"), + settings: settings, + deployment: deployment, }, mns: ns, } } func (b *manifest) labels() map[string]string { - return AppendLeaseLabels(b.lid, b.builder.labels()) + return AppendLeaseLabels(b.deployment.LeaseID(), b.builder.labels()) } func (b *manifest) Create() (*crd.Manifest, error) { - obj, err := crd.NewManifest(b.mns, b.lid, b.group) + obj, err := crd.NewManifest(b.mns, b.deployment.LeaseID(), b.deployment.ManifestGroup(), b.deployment.ClusterParams()) if err != nil { return nil, err @@ -52,7 +48,7 @@ func (b *manifest) Create() (*crd.Manifest, error) { } func (b *manifest) Update(obj *crd.Manifest) (*crd.Manifest, error) { - m, err := crd.NewManifest(b.mns, b.lid, b.group) + m, err := crd.NewManifest(b.mns, b.deployment.LeaseID(), b.deployment.ManifestGroup(), b.deployment.ClusterParams()) if err != nil { return nil, err } diff --git a/cluster/kube/builder/namespace.go b/cluster/kube/builder/namespace.go index c0629d765..af4079649 100644 --- a/cluster/kube/builder/namespace.go +++ b/cluster/kube/builder/namespace.go @@ -1,11 +1,8 @@ package builder import ( - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" ) type NS interface { @@ -20,12 +17,12 @@ type ns struct { var _ NS = (*ns)(nil) -func BuildNS(settings Settings, lid mtypes.LeaseID, group *mani.Group) NS { - return &ns{builder: builder{settings: settings, lid: lid, group: group}} +func BuildNS(settings Settings, deployment IClusterDeployment) NS { + return &ns{builder: builder{settings: settings, deployment: deployment}} } func (b *ns) labels() map[string]string { - return AppendLeaseLabels(b.lid, b.builder.labels()) + return AppendLeaseLabels(b.deployment.LeaseID(), b.builder.labels()) } func (b *ns) Create() (*corev1.Namespace, error) { // nolint:golint,unparam diff --git a/cluster/kube/builder/netpol.go b/cluster/kube/builder/netpol.go index db3372494..b62bbac8b 100644 --- a/cluster/kube/builder/netpol.go +++ b/cluster/kube/builder/netpol.go @@ -8,9 +8,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" manitypes "github.com/akash-network/akash-api/go/manifest/v2beta2" - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" sdlutil "github.com/akash-network/node/sdl/util" ) @@ -26,8 +24,8 @@ type netPol struct { var _ NetPol = (*netPol)(nil) -func BuildNetPol(settings Settings, lid mtypes.LeaseID, group *mani.Group) NetPol { - return &netPol{builder: builder{settings: settings, lid: lid, group: group}} +func BuildNetPol(settings Settings, deployment IClusterDeployment) NetPol { + return &netPol{builder: builder{settings: settings, deployment: deployment}} } // Create a set of NetworkPolicies to restrict the ingress traffic to a Tenant's @@ -46,7 +44,7 @@ func (b *netPol) Create() ([]*netv1.NetworkPolicy, error) { // nolint:golint,unp ObjectMeta: metav1.ObjectMeta{ Name: akashDeploymentPolicyName, Labels: b.labels(), - Namespace: LidNS(b.lid), + Namespace: LidNS(b.deployment.LeaseID()), }, Spec: netv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{}, @@ -60,7 +58,7 @@ func (b *netPol) Create() ([]*netv1.NetworkPolicy, error) { // nolint:golint,unp { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - akashNetworkNamespace: LidNS(b.lid), + akashNetworkNamespace: LidNS(b.deployment.LeaseID()), }, }, }, @@ -89,7 +87,7 @@ func (b *netPol) Create() ([]*netv1.NetworkPolicy, error) { // nolint:golint,unp { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - akashNetworkNamespace: LidNS(b.lid), + akashNetworkNamespace: LidNS(b.deployment.LeaseID()), }, }, }, @@ -142,7 +140,7 @@ func (b *netPol) Create() ([]*netv1.NetworkPolicy, error) { // nolint:golint,unp }, } - for _, service := range b.group.Services { + for _, service := range b.deployment.ManifestGroup().Services { // find all the ports that are exposed directly ports := make([]netv1.NetworkPolicyPort, 0) portsWithIP := make([]netv1.NetworkPolicyPort, 0) @@ -183,7 +181,7 @@ func (b *netPol) Create() ([]*netv1.NetworkPolicy, error) { // nolint:golint,unp ObjectMeta: metav1.ObjectMeta{ Labels: b.labels(), Name: policyName, - Namespace: LidNS(b.lid), + Namespace: LidNS(b.deployment.LeaseID()), }, Spec: netv1.NetworkPolicySpec{ Ingress: []netv1.NetworkPolicyIngressRule{ @@ -211,7 +209,7 @@ func (b *netPol) Create() ([]*netv1.NetworkPolicy, error) { // nolint:golint,unp ObjectMeta: metav1.ObjectMeta{ Labels: b.labels(), Name: policyName, - Namespace: LidNS(b.lid), + Namespace: LidNS(b.deployment.LeaseID()), }, Spec: netv1.NetworkPolicySpec{ Ingress: []netv1.NetworkPolicyIngressRule{ diff --git a/cluster/kube/builder/params_lease.go b/cluster/kube/builder/params_lease.go deleted file mode 100644 index 549e478db..000000000 --- a/cluster/kube/builder/params_lease.go +++ /dev/null @@ -1,67 +0,0 @@ -package builder - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/tendermint/tendermint/libs/log" - - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" - - crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" -) - -type ParamsServices interface { - builderBase - Create() (*crd.LeaseParamsService, error) - Update(obj *crd.LeaseParamsService) (*crd.LeaseParamsService, error) - Name() string -} - -type paramsServices struct { - builder - ns string -} - -var _ ParamsServices = (*paramsServices)(nil) - -func BuildParamsLeases(log log.Logger, settings Settings, ns string, lid mtypes.LeaseID, params crd.ParamsServices) ParamsServices { - return ¶msServices{ - builder: builder{ - log: log.With("module", "kube-builder"), - settings: settings, - lid: lid, - sparams: params, - }, - ns: ns, - } -} - -func (b *paramsServices) labels() map[string]string { - return AppendLeaseLabels(b.lid, b.builder.labels()) -} - -func (b *paramsServices) Create() (*crd.LeaseParamsService, error) { - obj := &crd.LeaseParamsService{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.ns, - Name: b.Name(), - Labels: b.labels(), - }, - Spec: crd.LeaseParamsServiceSpec{ - Services: b.sparams, - }, - } - - return obj, nil -} - -func (b *paramsServices) Update(obj *crd.LeaseParamsService) (*crd.LeaseParamsService, error) { - cobj := *obj - cobj.Spec.Services = b.sparams - - return &cobj, nil -} - -func (b *paramsServices) NS() string { - return b.ns -} diff --git a/cluster/kube/builder/podsecuritypolicy.go b/cluster/kube/builder/podsecuritypolicy.go index fb5193d9f..4000df0be 100644 --- a/cluster/kube/builder/podsecuritypolicy.go +++ b/cluster/kube/builder/podsecuritypolicy.go @@ -1,101 +1,90 @@ package builder -import ( - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" - corev1 "k8s.io/api/core/v1" - "k8s.io/api/policy/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" - - crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" -) - -type PspRestricted interface { - builderBase - Name() string - Create() (*v1beta1.PodSecurityPolicy, error) - Update(obj *v1beta1.PodSecurityPolicy) (*v1beta1.PodSecurityPolicy, error) -} - -type pspRestricted struct { - builder -} - -func BuildPSP( - settings Settings, - lid mtypes.LeaseID, - group *mani.Group, - sparams crd.ParamsServices) PspRestricted { // nolint:golint,unparam - return &pspRestricted{ - builder: builder{ - settings: settings, - lid: lid, - group: group, - sparams: sparams, - }, - } -} - -func (p *pspRestricted) Name() string { - return p.NS() -} - -func (p *pspRestricted) Create() (*v1beta1.PodSecurityPolicy, error) { // nolint:golint,unparam - falseVal := false - return &v1beta1.PodSecurityPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: p.Name(), - Namespace: p.Name(), - Labels: p.labels(), - Annotations: map[string]string{ - "seccomp.security.alpha.kubernetes.io/allowedProfileNames": "docker/default,runtime/default", - "apparmor.security.beta.kubernetes.io/allowedProfileNames": "runtime/default", - "seccomp.security.alpha.kubernetes.io/defaultProfileName": "runtime/default", - "apparmor.security.beta.kubernetes.io/defaultProfileName": "runtime/default", - }, - }, - Spec: v1beta1.PodSecurityPolicySpec{ - Privileged: false, - AllowPrivilegeEscalation: &falseVal, - RequiredDropCapabilities: []corev1.Capability{ - "ALL", - }, - Volumes: []v1beta1.FSType{ - v1beta1.EmptyDir, - v1beta1.PersistentVolumeClaim, // evaluate necessity later - }, - HostNetwork: false, - HostIPC: false, - HostPID: false, - RunAsUser: v1beta1.RunAsUserStrategyOptions{ - // fixme(#946): previous value RunAsUserStrategyMustRunAsNonRoot was interfering with - // (b *deployment) create() RunAsNonRoot: false - // allow any user at this moment till revise all security debris of kube api - Rule: v1beta1.RunAsUserStrategyRunAsAny, - }, - SELinux: v1beta1.SELinuxStrategyOptions{ - Rule: v1beta1.SELinuxStrategyRunAsAny, - }, - SupplementalGroups: v1beta1.SupplementalGroupsStrategyOptions{ - Rule: v1beta1.SupplementalGroupsStrategyRunAsAny, - }, - FSGroup: v1beta1.FSGroupStrategyOptions{ - Rule: v1beta1.FSGroupStrategyMustRunAs, - Ranges: []v1beta1.IDRange{ - { - Min: int64(1), - Max: int64(65535), - }, - }, - }, - ReadOnlyRootFilesystem: false, - }, - }, nil -} - -func (p *pspRestricted) Update(obj *v1beta1.PodSecurityPolicy) (*v1beta1.PodSecurityPolicy, error) { // nolint:golint,unparam - obj.Name = p.Name() - obj.Labels = p.labels() - return obj, nil -} +// type PspRestricted interface { +// builderBase +// Name() string +// Create() (*v1beta1.PodSecurityPolicy, error) +// Update(obj *v1beta1.PodSecurityPolicy) (*v1beta1.PodSecurityPolicy, error) +// } +// +// type pspRestricted struct { +// builder +// } +// +// func BuildPSP( +// settings Settings, +// lid mtypes.LeaseID, +// group *mani.Group, +// sparams crd.ParamsServices) PspRestricted { // nolint:golint,unparam +// return &pspRestricted{ +// builder: builder{ +// settings: settings, +// lid: lid, +// group: group, +// sparams: sparams, +// }, +// } +// } +// +// func (p *pspRestricted) Name() string { +// return p.NS() +// } +// +// func (p *pspRestricted) Create() (*v1beta1.PodSecurityPolicy, error) { // nolint:golint,unparam +// falseVal := false +// return &v1beta1.PodSecurityPolicy{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: p.Name(), +// Namespace: p.Name(), +// Labels: p.labels(), +// Annotations: map[string]string{ +// "seccomp.security.alpha.kubernetes.io/allowedProfileNames": "docker/default,runtime/default", +// "apparmor.security.beta.kubernetes.io/allowedProfileNames": "runtime/default", +// "seccomp.security.alpha.kubernetes.io/defaultProfileName": "runtime/default", +// "apparmor.security.beta.kubernetes.io/defaultProfileName": "runtime/default", +// }, +// }, +// Spec: v1beta1.PodSecurityPolicySpec{ +// Privileged: false, +// AllowPrivilegeEscalation: &falseVal, +// RequiredDropCapabilities: []corev1.Capability{ +// "ALL", +// }, +// Volumes: []v1beta1.FSType{ +// v1beta1.EmptyDir, +// v1beta1.PersistentVolumeClaim, // evaluate necessity later +// }, +// HostNetwork: false, +// HostIPC: false, +// HostPID: false, +// RunAsUser: v1beta1.RunAsUserStrategyOptions{ +// // fixme(#946): previous value RunAsUserStrategyMustRunAsNonRoot was interfering with +// // (b *deployment) create() RunAsNonRoot: false +// // allow any user at this moment till revise all security debris of kube api +// Rule: v1beta1.RunAsUserStrategyRunAsAny, +// }, +// SELinux: v1beta1.SELinuxStrategyOptions{ +// Rule: v1beta1.SELinuxStrategyRunAsAny, +// }, +// SupplementalGroups: v1beta1.SupplementalGroupsStrategyOptions{ +// Rule: v1beta1.SupplementalGroupsStrategyRunAsAny, +// }, +// FSGroup: v1beta1.FSGroupStrategyOptions{ +// Rule: v1beta1.FSGroupStrategyMustRunAs, +// Ranges: []v1beta1.IDRange{ +// { +// Min: int64(1), +// Max: int64(65535), +// }, +// }, +// }, +// ReadOnlyRootFilesystem: false, +// }, +// }, nil +// } +// +// func (p *pspRestricted) Update(obj *v1beta1.PodSecurityPolicy) (*v1beta1.PodSecurityPolicy, error) { // nolint:golint,unparam +// obj.Name = p.Name() +// obj.Labels = p.labels() +// return obj, nil +// } diff --git a/cluster/kube/builder/service.go b/cluster/kube/builder/service.go index fee97c0bd..e641281c9 100644 --- a/cluster/kube/builder/service.go +++ b/cluster/kube/builder/service.go @@ -4,15 +4,11 @@ import ( "fmt" "github.com/pkg/errors" - "github.com/tendermint/tendermint/libs/log" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" manitypes "github.com/akash-network/akash-api/go/manifest/v2beta2" - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" - sdlutil "github.com/akash-network/node/sdl/util" ) @@ -24,35 +20,25 @@ type Service interface { } type service struct { - workload + Workload requireNodePort bool } var _ Service = (*service)(nil) -func BuildService( - log log.Logger, - settings Settings, - lid mtypes.LeaseID, - group *mani.Group, - serviceIdx int, - requireNodePort bool) Service { - return &service{ - workload: workload{ - builder: builder{ - settings: settings, - log: log.With("module", "kube-builder"), - lid: lid, - group: group, - }, - serviceIdx: serviceIdx, - }, +func BuildService(workload Workload, requireNodePort bool) Service { + ss := &service{ + Workload: workload, requireNodePort: requireNodePort, } + + ss.Workload.log = ss.Workload.log.With("object", "service", "service-name", ss.deployment.ManifestGroup().Services[ss.serviceIdx].Name) + + return ss } func (b *service) Name() string { - basename := b.workload.Name() + basename := b.Workload.Name() if b.requireNodePort { return makeGlobalServiceNameFromBasename(basename) } @@ -120,7 +106,7 @@ func (b *service) Update(obj *corev1.Service) (*corev1.Service, error) { // noli } func (b *service) Any() bool { - service := &b.group.Services[b.serviceIdx] + service := &b.deployment.ManifestGroup().Services[b.serviceIdx] for _, expose := range service.Expose { exposeIsIngress := sdlutil.ShouldBeIngress(expose) @@ -143,7 +129,7 @@ var errUnsupportedProtocol = errors.New("Unsupported protocol for service") var errInvalidServiceBuilder = errors.New("service builder invalid") func (b *service) ports() ([]corev1.ServicePort, error) { - service := &b.group.Services[b.serviceIdx] + service := &b.deployment.ManifestGroup().Services[b.serviceIdx] ports := make([]corev1.ServicePort, 0, len(service.Expose)) portsAdded := make(map[int32]struct{}) diff --git a/cluster/kube/builder/settings.go b/cluster/kube/builder/settings.go index 40c18ab38..4f063ed51 100644 --- a/cluster/kube/builder/settings.go +++ b/cluster/kube/builder/settings.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - validation_util "github.com/akash-network/node/util/validation" + vutil "github.com/akash-network/node/util/validation" ) // Settings configures k8s object generation such that it is customized to the @@ -53,7 +53,7 @@ func ValidateSettings(settings Settings) error { return errors.Wrap(ErrSettingsValidation, "empty ingress domain") } - if !validation_util.IsDomainName(settings.DeploymentIngressDomain) { + if !vutil.IsDomainName(settings.DeploymentIngressDomain) { return fmt.Errorf("%w: invalid domain name %q", ErrSettingsValidation, settings.DeploymentIngressDomain) } } diff --git a/cluster/kube/builder/statefulset.go b/cluster/kube/builder/statefulset.go index 3c52f7b03..c5fcda50f 100644 --- a/cluster/kube/builder/statefulset.go +++ b/cluster/kube/builder/statefulset.go @@ -1,15 +1,9 @@ package builder import ( - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" - "github.com/tendermint/tendermint/libs/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" - - crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) type StatefulSet interface { @@ -19,39 +13,24 @@ type StatefulSet interface { } type statefulSet struct { - workload + Workload } var _ StatefulSet = (*statefulSet)(nil) -func BuildStatefulSet( - log log.Logger, - settings Settings, - lid mtypes.LeaseID, - group *mani.Group, - sparams crd.ParamsServices, - serviceIdx int) StatefulSet { - return &statefulSet{ - workload: newWorkloadBuilder(log, settings, lid, group, sparams, serviceIdx), +func BuildStatefulSet(workload Workload) StatefulSet { + ss := &statefulSet{ + Workload: workload, } + + ss.Workload.log = ss.Workload.log.With("object", "statefulset", "service-name", ss.deployment.ManifestGroup().Services[ss.serviceIdx].Name) + + return ss } func (b *statefulSet) Create() (*appsv1.StatefulSet, error) { // nolint:golint,unparam - service := &b.group.Services[b.serviceIdx] - params := &b.sparams[b.serviceIdx].Params - - replicas := int32(service.Count) falseValue := false - // fixme b.runtimeClassName is updated on call to the container() - containers := []corev1.Container{b.container()} - - var effectiveRuntimeClassName *string - if len(params.RuntimeClass) != 0 && params.RuntimeClass != runtimeClassNoneValue { - runtimeClass := params.RuntimeClass - effectiveRuntimeClassName = &runtimeClass - } - kdeployment := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: b.Name(), @@ -61,18 +40,19 @@ func (b *statefulSet) Create() (*appsv1.StatefulSet, error) { // nolint:golint,u Selector: &metav1.LabelSelector{ MatchLabels: b.labels(), }, - Replicas: &replicas, + Replicas: b.replicas(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: b.labels(), }, Spec: corev1.PodSpec{ - RuntimeClassName: effectiveRuntimeClassName, + Affinity: b.affinity(), + RuntimeClassName: b.runtimeClass(), SecurityContext: &corev1.PodSecurityContext{ RunAsNonRoot: &falseValue, }, AutomountServiceAccountToken: &falseValue, - Containers: containers, + Containers: []corev1.Container{b.container()}, ImagePullSecrets: b.imagePullSecrets(), }, }, @@ -84,13 +64,12 @@ func (b *statefulSet) Create() (*appsv1.StatefulSet, error) { // nolint:golint,u } func (b *statefulSet) Update(obj *appsv1.StatefulSet) (*appsv1.StatefulSet, error) { // nolint:golint,unparam - service := &b.group.Services[b.serviceIdx] - replicas := int32(service.Count) - obj.Labels = b.labels() obj.Spec.Selector.MatchLabels = b.labels() - obj.Spec.Replicas = &replicas + obj.Spec.Replicas = b.replicas() obj.Spec.Template.Labels = b.labels() + obj.Spec.Template.Spec.Affinity = b.affinity() + obj.Spec.Template.Spec.RuntimeClassName = b.runtimeClass() obj.Spec.Template.Spec.Containers = []corev1.Container{b.container()} obj.Spec.Template.Spec.ImagePullSecrets = b.imagePullSecrets() obj.Spec.VolumeClaimTemplates = b.persistentVolumeClaims() diff --git a/cluster/kube/builder/workload.go b/cluster/kube/builder/workload.go index 41e51ca0c..2d2628804 100644 --- a/cluster/kube/builder/workload.go +++ b/cluster/kube/builder/workload.go @@ -4,13 +4,11 @@ import ( "fmt" "strings" - mani "github.com/akash-network/akash-api/go/manifest/v2beta2" - "github.com/tendermint/tendermint/libs/log" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" + "github.com/tendermint/tendermint/libs/log" "github.com/akash-network/node/sdl" sdlutil "github.com/akash-network/node/sdl/util" @@ -19,7 +17,10 @@ import ( ) const ( - ResourceNvidiaGPU = corev1.ResourceName("nvidia.com/gpu") + ResourceGPUNvidia = corev1.ResourceName("nvidia.com/gpu") + ResourceGPUAMD = corev1.ResourceName("amd.com/gpu") + GPUVendorNvidia = "nvidia" + GPUVendorAMD = "amd" ) type workloadBase interface { @@ -27,36 +28,37 @@ type workloadBase interface { Name() string } -type workload struct { +type Workload struct { builder serviceIdx int } -var _ workloadBase = (*workload)(nil) +var _ workloadBase = (*Workload)(nil) -func newWorkloadBuilder( +func NewWorkloadBuilder( log log.Logger, settings Settings, - lid mtypes.LeaseID, - group *mani.Group, - sparams crd.ParamsServices, - serviceIdx int) workload { - return workload{ + deployment IClusterDeployment, + serviceIdx int) Workload { + return Workload{ builder: builder{ - settings: settings, - log: log.With("module", "kube-builder"), - lid: lid, - group: group, - sparams: sparams, + settings: settings, + log: log.With("module", "kube-builder"), + deployment: deployment, }, serviceIdx: serviceIdx, } } -func (b *workload) container() corev1.Container { +func (b *Workload) Name() string { + return b.deployment.ManifestGroup().Services[b.serviceIdx].Name +} + +func (b *Workload) container() corev1.Container { falseValue := false - service := &b.group.Services[b.serviceIdx] + service := &b.deployment.ManifestGroup().Services[b.serviceIdx] + sparams := b.deployment.ClusterParams().SchedulerParams[b.serviceIdx] kcontainer := corev1.Container{ Name: service.Name, @@ -82,13 +84,22 @@ func (b *workload) container() corev1.Container { } if gpu := service.Resources.GPU; gpu != nil && gpu.Units.Value() > 0 { - requestedGPU := sdlutil.ComputeCommittedResources(b.settings.GPUCommitLevel, gpu.Units) + var resourceName corev1.ResourceName + + switch sparams.Resources.GPU.Vendor { + case GPUVendorNvidia: + resourceName = ResourceGPUNvidia + case GPUVendorAMD: + resourceName = ResourceGPUAMD + default: + panic(fmt.Sprintf("requested for unsupported GPU vendor")) + } + // GPUs are only supposed to be specified in the limits section, which means // - can specify GPU limits without specifying requests, because Kubernetes will use the limit as the request value by default. // - can specify GPU in both limits and requests but these two values must be equal. // - cannot specify GPU requests without specifying limits. - // fixme get custom resource name from inventory - resourceName := ResourceNvidiaGPU + requestedGPU := sdlutil.ComputeCommittedResources(b.settings.GPUCommitLevel, gpu.Units) kcontainer.Resources.Requests[resourceName] = resource.NewQuantity(int64(requestedGPU.Value()), resource.DecimalSI).DeepCopy() kcontainer.Resources.Limits[resourceName] = resource.NewQuantity(int64(gpu.Units.Value()), resource.DecimalSI).DeepCopy() } @@ -143,10 +154,10 @@ func (b *workload) container() corev1.Container { return kcontainer } -func (b *workload) persistentVolumeClaims() []corev1.PersistentVolumeClaim { +func (b *Workload) persistentVolumeClaims() []corev1.PersistentVolumeClaim { var pvcs []corev1.PersistentVolumeClaim // nolint:prealloc - service := &b.group.Services[b.serviceIdx] + service := &b.deployment.ManifestGroup().Services[b.serviceIdx] for _, storage := range service.Resources.Storage { attr := storage.Attributes.Find(sdl.StorageAttributePersistent) @@ -184,17 +195,82 @@ func (b *workload) persistentVolumeClaims() []corev1.PersistentVolumeClaim { return pvcs } -func (b *workload) Name() string { - return b.group.Services[b.serviceIdx].Name +func (b *Workload) runtimeClass() *string { + params := b.deployment.ClusterParams().SchedulerParams[b.serviceIdx] + var effectiveRuntimeClassName *string + + if params != nil { + if len(params.RuntimeClass) != 0 && params.RuntimeClass != runtimeClassNoneValue { + runtimeClass := new(string) + *runtimeClass = params.RuntimeClass + effectiveRuntimeClassName = runtimeClass + } + } + + return effectiveRuntimeClassName } -func (b *workload) labels() map[string]string { +func (b *Workload) replicas() *int32 { + replicas := new(int32) + *replicas = int32(b.deployment.ManifestGroup().Services[b.serviceIdx].Count) + + return replicas +} + +func (b *Workload) affinity() *corev1.Affinity { + svc := b.deployment.ClusterParams().SchedulerParams[b.serviceIdx] + + if svc == nil || svc.Resources == nil { + return nil + } + + selectors := nodeSelectorsFromResources(svc.Resources) + if len(selectors) == 0 { + return nil + } + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: selectors, + }, + }, + }, + }, + } + + return affinity +} + +func nodeSelectorsFromResources(res *crd.SchedulerResources) []corev1.NodeSelectorRequirement { + if res == nil { + return nil + } + + var selectors []corev1.NodeSelectorRequirement + + if gpu := res.GPU; gpu != nil { + selectors = append(selectors, corev1.NodeSelectorRequirement{ + Key: fmt.Sprintf("%s/vendor/%s/model/%s", AkashServiceCapabilityGPU, gpu.Vendor, gpu.Model), + Operator: "In", + Values: []string{ + "true", + }, + }) + } + + return selectors +} + +func (b *Workload) labels() map[string]string { obj := b.builder.labels() - obj[AkashManifestServiceLabelName] = b.group.Services[b.serviceIdx].Name + obj[AkashManifestServiceLabelName] = b.deployment.ManifestGroup().Services[b.serviceIdx].Name return obj } -func (b *workload) imagePullSecrets() []corev1.LocalObjectReference { +func (b *Workload) imagePullSecrets() []corev1.LocalObjectReference { if b.settings.DockerImagePullSecretsName == "" { return nil } @@ -202,13 +278,16 @@ func (b *workload) imagePullSecrets() []corev1.LocalObjectReference { return []corev1.LocalObjectReference{{Name: b.settings.DockerImagePullSecretsName}} } -func (b *workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar { +func (b *Workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar { + lid := b.deployment.LeaseID() + // Add each env. var. if it is not already set by the SDL - env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashGroupSequence, b.lid.GetGSeq()) - env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashDeploymentSequence, b.lid.GetDSeq()) - env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashOrderSequence, b.lid.GetOSeq()) - env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashOwner, b.lid.Owner) - env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashProvider, b.lid.Provider) + env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashGroupSequence, lid.GetGSeq()) + env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashDeploymentSequence, lid.GetDSeq()) + env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashOrderSequence, lid.GetOSeq()) + env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashOwner, lid.Owner) + env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashProvider, lid.Provider) env = addIfNotPresent(envVarsAlreadyAdded, env, envVarAkashClusterPublicHostname, b.settings.ClusterPublicHostname) + return env } diff --git a/cluster/kube/client.go b/cluster/kube/client.go index 1369a6e08..38b8e75c4 100644 --- a/cluster/kube/client.go +++ b/cluster/kube/client.go @@ -3,12 +3,14 @@ package kube import ( "context" "fmt" + "io" "strings" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/tendermint/tendermint/libs/log" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" kubeErrors "k8s.io/apimachinery/pkg/api/errors" @@ -62,6 +64,18 @@ func (c *client) String() string { return fmt.Sprintf("kube client %p ns=%s", c, c.ns) } +func wrapKubeCall[T any](label string, fn func() (T, error)) (T, error) { + res, err := fn() + + status := metricsutils.SuccessLabel + if err != nil { + status = metricsutils.FailLabel + } + kubeCallsCounter.WithLabelValues(label, status).Inc() + + return res, err +} + // NewClient returns new Kubernetes Client instance with provided logger, host and ns. Returns error in-case of failure // configPath may be the empty string func NewClient(ctx context.Context, log log.Logger, ns string, configPath string) (Client, error) { @@ -101,29 +115,31 @@ func NewClient(ctx context.Context, log log.Logger, ns string, configPath string }, nil } -func (c *client) GetDeployments(ctx context.Context, dID dtypes.DeploymentID) ([]ctypes.Deployment, error) { +func (c *client) GetDeployments(ctx context.Context, dID dtypes.DeploymentID) ([]ctypes.IDeployment, error) { labelSelectors := &strings.Builder{} _, _ = fmt.Fprintf(labelSelectors, "%s=%d", builder.AkashLeaseDSeqLabelName, dID.DSeq) _, _ = fmt.Fprintf(labelSelectors, ",%s=%s", builder.AkashLeaseOwnerLabelName, dID.Owner) - manifests, err := c.ac.AkashV2beta2().Manifests(c.ns).List(ctx, metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{}, - LabelSelector: labelSelectors.String(), - FieldSelector: "", - Watch: false, - AllowWatchBookmarks: false, - ResourceVersion: "", - ResourceVersionMatch: "", - TimeoutSeconds: nil, - Limit: 0, - Continue: "", + manifests, err := wrapKubeCall("manifests-list", func() (*crd.ManifestList, error) { + return c.ac.AkashV2beta2().Manifests(c.ns).List(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{}, + LabelSelector: labelSelectors.String(), + FieldSelector: "", + Watch: false, + AllowWatchBookmarks: false, + ResourceVersion: "", + ResourceVersionMatch: "", + TimeoutSeconds: nil, + Limit: 0, + Continue: "", + }) }) if err != nil { return nil, err } - result := make([]ctypes.Deployment, len(manifests.Items)) + result := make([]ctypes.IDeployment, len(manifests.Items)) for i, manifest := range manifests.Items { result[i], err = manifest.Deployment() if err != nil { @@ -137,7 +153,10 @@ func (c *client) GetDeployments(ctx context.Context, dID dtypes.DeploymentID) ([ func (c *client) GetManifestGroup(ctx context.Context, lID mtypes.LeaseID) (bool, crd.ManifestGroup, error) { leaseNamespace := builder.LidNS(lID) - obj, err := c.ac.AkashV2beta2().Manifests(c.ns).Get(ctx, leaseNamespace, metav1.GetOptions{}) + obj, err := wrapKubeCall("manifests-get", func() (*crd.Manifest, error) { + return c.ac.AkashV2beta2().Manifests(c.ns).Get(ctx, leaseNamespace, metav1.GetOptions{}) + }) + if err != nil { if kubeErrors.IsNotFound(err) { c.log.Info("CRD manifest not found", "lease-ns", leaseNamespace) @@ -150,13 +169,16 @@ func (c *client) GetManifestGroup(ctx context.Context, lID mtypes.LeaseID) (bool return true, obj.Spec.Group, nil } -func (c *client) Deployments(ctx context.Context) ([]ctypes.Deployment, error) { - manifests, err := c.ac.AkashV2beta2().Manifests(c.ns).List(ctx, metav1.ListOptions{}) +func (c *client) Deployments(ctx context.Context) ([]ctypes.IDeployment, error) { + manifests, err := wrapKubeCall("manifests-list", func() (*crd.ManifestList, error) { + return c.ac.AkashV2beta2().Manifests(c.ns).List(ctx, metav1.ListOptions{}) + }) + if err != nil { return nil, err } - deployments := make([]ctypes.Deployment, 0, len(manifests.Items)) + deployments := make([]ctypes.IDeployment, 0, len(manifests.Items)) for _, manifest := range manifests.Items { deployment, err := manifest.Deployment() if err != nil { @@ -168,7 +190,15 @@ func (c *client) Deployments(ctx context.Context) ([]ctypes.Deployment, error) { return deployments, nil } -func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Group) error { +func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) error { + cdeployment, err := builder.ClusterDeploymentFromDeployment(deployment) + if err != nil { + return err + } + + lid := cdeployment.LeaseID() + group := cdeployment.ManifestGroup() + settingsI := ctx.Value(builder.SettingsKey) if nil == settingsI { return kubeclienterrors.ErrNotConfiguredWithSettings @@ -178,34 +208,18 @@ func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Gro return err } - if err := applyNS(ctx, c.kc, builder.BuildNS(settings, lid, group)); err != nil { + if err := applyNS(ctx, c.kc, builder.BuildNS(settings, cdeployment)); err != nil { c.log.Error("applying namespace", "err", err, "lease", lid) return err } - if err := applyNetPolicies(ctx, c.kc, builder.BuildNetPol(settings, lid, group)); err != nil { // + if err := applyNetPolicies(ctx, c.kc, builder.BuildNetPol(settings, cdeployment)); err != nil { // c.log.Error("applying namespace network policies", "err", err, "lease", lid) return err } - if err := applyManifest(ctx, c.ac, builder.BuildManifest(c.log, settings, c.ns, lid, group)); err != nil { - c.log.Error("applying manifest", "err", err, "lease", lid) - return err - } - - params := make(crd.ParamsServices, 0, len(group.Services)) - - for _, svc := range group.Services { - sParams := crd.ParamsService{} - - if svc.Resources.GPU.Units.Value() > 0 { - sParams.Params.RuntimeClass = "nvidia" - } - - params = append(params, sParams) - } - - if err := applyServicesParams(ctx, c.ac, builder.BuildParamsLeases(c.log, settings, c.ns, lid, params)); err != nil { + cmanifest := builder.BuildManifest(c.log, settings, c.ns, cdeployment) + if err := applyManifest(ctx, c.ac, cmanifest); err != nil { c.log.Error("applying manifest", "err", err, "lease", lid) return err } @@ -216,6 +230,8 @@ func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Gro } for svcIdx := range group.Services { + workload := builder.NewWorkloadBuilder(c.log, settings, cdeployment, svcIdx) + service := &group.Services[svcIdx] persistent := false @@ -227,12 +243,12 @@ func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Gro } if persistent { - if err := applyStatefulSet(ctx, c.kc, builder.BuildStatefulSet(c.log, settings, lid, group, params, svcIdx)); err != nil { + if err := applyStatefulSet(ctx, c.kc, builder.BuildStatefulSet(workload)); err != nil { c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name) return err } } else { - if err := applyDeployment(ctx, c.kc, builder.NewDeployment(c.log, settings, lid, group, params, svcIdx)); err != nil { + if err := applyDeployment(ctx, c.kc, builder.NewDeployment(workload)); err != nil { c.log.Error("applying deployment", "err", err, "lease", lid, "service", service.Name) return err } @@ -243,7 +259,7 @@ func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Gro continue } - serviceBuilderLocal := builder.BuildService(c.log, settings, lid, group, svcIdx, false) + serviceBuilderLocal := builder.BuildService(workload, false) if serviceBuilderLocal.Any() { if err := applyService(ctx, c.kc, serviceBuilderLocal); err != nil { c.log.Error("applying local service", "err", err, "lease", lid, "service", service.Name) @@ -251,7 +267,7 @@ func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Gro } } - serviceBuilderGlobal := builder.BuildService(c.log, settings, lid, group, svcIdx, true) + serviceBuilderGlobal := builder.BuildService(workload, true) if serviceBuilderGlobal.Any() { if err := applyService(ctx, c.kc, serviceBuilderGlobal); err != nil { c.log.Error("applying global service", "err", err, "lease", lid, "service", service.Name) @@ -264,15 +280,14 @@ func (c *client) Deploy(ctx context.Context, lid mtypes.LeaseID, group *mapi.Gro } func (c *client) TeardownLease(ctx context.Context, lid mtypes.LeaseID) error { - result := c.kc.CoreV1().Namespaces().Delete(ctx, builder.LidNS(lid), metav1.DeleteOptions{}) + _, result := wrapKubeCall("namespaces-delete", func() (interface{}, error) { + return nil, c.kc.CoreV1().Namespaces().Delete(ctx, builder.LidNS(lid), metav1.DeleteOptions{}) + }) - label := metricsutils.SuccessLabel - if result != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("namespaces-delete", label).Inc() + _, err := wrapKubeCall("manifests-delete", func() (interface{}, error) { + return nil, c.ac.AkashV2beta2().Manifests(c.ns).Delete(ctx, builder.LidNS(lid), metav1.DeleteOptions{}) + }) - err := c.ac.AkashV2beta2().Manifests(c.ns).Delete(ctx, builder.LidNS(lid), metav1.DeleteOptions{}) if err != nil { c.log.Error("teardown lease: unable to delete manifest", "ns", builder.LidNS(lid), "error", err) } @@ -346,24 +361,20 @@ func (c *client) LeaseEvents(ctx context.Context, lid mtypes.LeaseID, services s var wtch ctypes.EventsWatcher if follow { - watcher, err := c.kc.EventsV1().Events(builder.LidNS(lid)).Watch(ctx, listOpts) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("events-follow", label).Inc() + watcher, err := wrapKubeCall("events-follow", func() (watch.Interface, error) { + return c.kc.EventsV1().Events(builder.LidNS(lid)).Watch(ctx, listOpts) + }) + if err != nil { return nil, err } wtch = newEventsFeedWatch(ctx, watcher) } else { - list, err := c.kc.EventsV1().Events(builder.LidNS(lid)).List(ctx, listOpts) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("events-list", label).Inc() + list, err := wrapKubeCall("events-list", func() (*eventsv1.EventList, error) { + return c.kc.EventsV1().Events(builder.LidNS(lid)).List(ctx, listOpts) + }) + if err != nil { return nil, err } @@ -387,28 +398,23 @@ func (c *client) LeaseLogs(ctx context.Context, lid mtypes.LeaseID, c.log.Info("filtering pods", "labelSelector", listOpts.LabelSelector) - pods, err := c.kc.CoreV1().Pods(builder.LidNS(lid)).List(ctx, listOpts) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("pods-list", label).Inc() + pods, err := wrapKubeCall("pods-list", func() (*corev1.PodList, error) { + return c.kc.CoreV1().Pods(builder.LidNS(lid)).List(ctx, listOpts) + }) if err != nil { c.log.Error("listing pods", "err", err) return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) } streams := make([]*ctypes.ServiceLog, len(pods.Items)) for i, pod := range pods.Items { - stream, err := c.kc.CoreV1().Pods(builder.LidNS(lid)).GetLogs(pod.Name, &corev1.PodLogOptions{ - Follow: follow, - TailLines: tailLines, - Timestamps: false, - }).Stream(ctx) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("pods-getlogs", label).Inc() + stream, err := wrapKubeCall("pods-getlogs", func() (io.ReadCloser, error) { + return c.kc.CoreV1().Pods(builder.LidNS(lid)).GetLogs(pod.Name, &corev1.PodLogOptions{ + Follow: follow, + TailLines: tailLines, + Timestamps: false, + }).Stream(ctx) + }) + if err != nil { c.log.Error("get pod logs", "err", err) return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) @@ -428,13 +434,9 @@ func (c *client) ForwardedPortStatus(ctx context.Context, leaseID mtypes.LeaseID return nil, err } - services, err := c.kc.CoreV1().Services(builder.LidNS(leaseID)).List(ctx, metav1.ListOptions{}) - - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("services-list", label).Inc() + services, err := wrapKubeCall("services-list", func() (*corev1.ServiceList, error) { + return c.kc.CoreV1().Services(builder.LidNS(leaseID)).List(ctx, metav1.ListOptions{}) + }) if err != nil { c.log.Error("list services", "err", err) return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) @@ -504,10 +506,11 @@ func (c *client) LeaseStatus(ctx context.Context, lid mtypes.LeaseID) (map[strin kubeSelectorForLease(labelSelector, lid) // Note: this is a separate call to the Kubernetes API to get this data. It could // be a separate method on the interface entirely - phResult, err := c.ac.AkashV2beta2().ProviderHosts(c.ns).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector.String(), + phResult, err := wrapKubeCall("providerhosts-list", func() (*crd.ProviderHostList, error) { + return c.ac.AkashV2beta2().ProviderHosts(c.ns).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector.String(), + }) }) - if err != nil { return nil, err } @@ -531,7 +534,10 @@ func (c *client) ServiceStatus(ctx context.Context, lid mtypes.LeaseID, name str // Get manifest definition from CRD c.log.Debug("Pulling manifest from CRD", "lease-ns", builder.LidNS(lid)) - mani, err := c.ac.AkashV2beta2().Manifests(c.ns).Get(ctx, builder.LidNS(lid), metav1.GetOptions{}) + mani, err := wrapKubeCall("manifests-list", func() (*crd.Manifest, error) { + return c.ac.AkashV2beta2().Manifests(c.ns).Get(ctx, builder.LidNS(lid), metav1.GetOptions{}) + }) + if err != nil { c.log.Error("CRD manifest not found", "lease-ns", builder.LidNS(lid), "name", name) return nil, kubeclienterrors.ErrNoManifestForLease @@ -556,12 +562,9 @@ func (c *client) ServiceStatus(ctx context.Context, lid mtypes.LeaseID, name str if isDeployment { c.log.Debug("get deployment", "lease-ns", builder.LidNS(lid), "name", name) - deployment, err := c.kc.AppsV1().Deployments(builder.LidNS(lid)).Get(ctx, name, metav1.GetOptions{}) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("deployments-get", label).Inc() + deployment, err := wrapKubeCall("deployments-get", func() (*appsv1.Deployment, error) { + return c.kc.AppsV1().Deployments(builder.LidNS(lid)).Get(ctx, name, metav1.GetOptions{}) + }) if err != nil { c.log.Error("deployment get", "err", err) @@ -584,12 +587,9 @@ func (c *client) ServiceStatus(ctx context.Context, lid mtypes.LeaseID, name str } } else { c.log.Debug("get statefulsets", "lease-ns", builder.LidNS(lid), "name", name) - statefulset, err := c.kc.AppsV1().StatefulSets(builder.LidNS(lid)).Get(ctx, name, metav1.GetOptions{}) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("statefulsets-get", label).Inc() + statefulset, err := wrapKubeCall("statefulsets-get", func() (*appsv1.StatefulSet, error) { + return c.kc.AppsV1().StatefulSets(builder.LidNS(lid)).Get(ctx, name, metav1.GetOptions{}) + }) if err != nil { c.log.Error("statefulsets get", "err", err) @@ -651,40 +651,33 @@ exposeCheckLoop: labelSelector := &strings.Builder{} kubeSelectorForLease(labelSelector, lid) - phs, err := c.ac.AkashV2beta2().ProviderHosts(c.ns).List(ctx, metav1.ListOptions{ - LabelSelector: labelSelector.String(), + phs, err := wrapKubeCall("provider-hosts", func() (*crd.ProviderHostList, error) { + return c.ac.AkashV2beta2().ProviderHosts(c.ns).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector.String(), + }) }) - if hasHostnames { - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("provider-hosts", label).Inc() - if err != nil { - c.log.Error("provider hosts get", "err", err) - return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) - } - - hosts := make([]string, 0, len(phs.Items)) - for _, ph := range phs.Items { - hosts = append(hosts, ph.Spec.Hostname) - } + if err != nil { + c.log.Error("provider hosts get", "err", err) + return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) + } - result.URIs = hosts + hosts := make([]string, 0, len(phs.Items)) + for _, ph := range phs.Items { + hosts = append(hosts, ph.Spec.Hostname) } + + result.URIs = hosts } return result, nil } func (c *client) leaseExists(ctx context.Context, lid mtypes.LeaseID) error { - _, err := c.kc.CoreV1().Namespaces().Get(ctx, builder.LidNS(lid), metav1.GetOptions{}) - label := metricsutils.SuccessLabel - if err != nil && !kubeErrors.IsNotFound(err) { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("namespace-get", label).Inc() + _, err := wrapKubeCall("namespace-get", func() (*corev1.Namespace, error) { + return c.kc.CoreV1().Namespaces().Get(ctx, builder.LidNS(lid), metav1.GetOptions{}) + }) + if err != nil { if kubeErrors.IsNotFound(err) { return kubeclienterrors.ErrLeaseNotFound @@ -702,23 +695,19 @@ func (c *client) deploymentsForLease(ctx context.Context, lid mtypes.LeaseID) (m return nil, err } - deployments, err := c.kc.AppsV1().Deployments(builder.LidNS(lid)).List(ctx, metav1.ListOptions{}) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("deployments-list", label).Inc() + deployments, err := wrapKubeCall("deployments-list", func() (*appsv1.DeploymentList, error) { + return c.kc.AppsV1().Deployments(builder.LidNS(lid)).List(ctx, metav1.ListOptions{}) + }) + if err != nil { c.log.Error("deployments list", "err", err) return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) } - statefulsets, err := c.kc.AppsV1().StatefulSets(builder.LidNS(lid)).List(ctx, metav1.ListOptions{}) - label = metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("statefulsets-list", label).Inc() + statefulsets, err := wrapKubeCall("statefulsets-list", func() (*appsv1.StatefulSetList, error) { + return c.kc.AppsV1().StatefulSets(builder.LidNS(lid)).List(ctx, metav1.ListOptions{}) + }) + if err != nil { c.log.Error("statefulsets list", "err", err) return nil, errors.Wrap(err, kubeclienterrors.ErrInternalError.Error()) @@ -765,5 +754,7 @@ func (c *client) deploymentsForLease(ctx context.Context, lid mtypes.LeaseID) (m } func (c *client) KubeVersion() (*version.Info, error) { - return c.kc.Discovery().ServerVersion() + return wrapKubeCall("discovery-serverversion", func() (*version.Info, error) { + return c.kc.Discovery().ServerVersion() + }) } diff --git a/cluster/kube/client_exec_test.go b/cluster/kube/client_exec_test.go index 0cba79668..8c48e18e5 100644 --- a/cluster/kube/client_exec_test.go +++ b/cluster/kube/client_exec_test.go @@ -107,7 +107,11 @@ func withExecTestScaffold(t *testing.T, changePod func(pod *corev1.Pod) error, t require.NoError(t, err) require.Len(t, mani, 1) - s.crdManifest, err = crd.NewManifest(testKubeClientNs, s.leaseID, &mani[0]) + sparams := crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(mani[0].Services)), + } + + s.crdManifest, err = crd.NewManifest(testKubeClientNs, s.leaseID, &mani[0], sparams) require.NoError(t, err) require.NotNil(t, s.crdManifest) diff --git a/cluster/kube/client_test.go b/cluster/kube/client_test.go index 83d60fbce..bf0895607 100644 --- a/cluster/kube/client_test.go +++ b/cluster/kube/client_test.go @@ -455,7 +455,7 @@ func TestServiceStatusNoServiceWithName(t *testing.T) { Services: nil, } - m, err := crd.NewManifest(testKubeClientNs, lid, mg) + m, err := crd.NewManifest(testKubeClientNs, lid, mg, crd.ClusterSettings{SchedulerParams: nil}) require.NoError(t, err) akashMock := akashclient_fake.NewSimpleClientset(m) @@ -495,7 +495,7 @@ func TestServiceStatusNoCRDManifest(t *testing.T) { Services: nil, } - m, err := crd.NewManifest(testKubeClientNs+"a", lid, mg) + m, err := crd.NewManifest(testKubeClientNs+"a", lid, mg, crd.ClusterSettings{SchedulerParams: nil}) require.NoError(t, err) akashMock := akashclient_fake.NewSimpleClientset(m) @@ -575,7 +575,11 @@ func TestServiceStatusWithIngress(t *testing.T) { Services: services, } - m, err := crd.NewManifest(testKubeClientNs, lid, mg) + cparams := crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(mg.Services)), + } + + m, err := crd.NewManifest(testKubeClientNs, lid, mg, cparams) require.NoError(t, err) akashMock := akashclient_fake.NewSimpleClientset(m, fakeProviderHost("abcd.com", lid, "echo", 9000)) @@ -730,7 +734,11 @@ func TestServiceStatusWithoutIngress(t *testing.T) { Services: services, } - m, err := crd.NewManifest(testKubeClientNs, lid, mg) + cparams := crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(mg.Services)), + } + + m, err := crd.NewManifest(testKubeClientNs, lid, mg, cparams) require.NoError(t, err) akashMock := akashclient_fake.NewSimpleClientset(m) diff --git a/cluster/kube/deploy_test.go b/cluster/kube/deploy_test.go index 4ebdae7bd..bf094ac92 100644 --- a/cluster/kube/deploy_test.go +++ b/cluster/kube/deploy_test.go @@ -14,6 +14,8 @@ import ( "github.com/akash-network/node/testutil" "github.com/akash-network/provider/cluster/kube/builder" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" + crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) const ( @@ -48,6 +50,14 @@ func TestDeploy(t *testing.T) { require.NoError(t, err) ctx = context.WithValue(ctx, builder.SettingsKey, builder.NewDefaultSettings()) - err = client.Deploy(ctx, leaseID, &mani.GetGroups()[0]) + group := &mani.GetGroups()[0] + cdep := &ctypes.Deployment{ + Lid: leaseID, + MGroup: group, + CParams: crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(group.Services)), + }, + } + err = client.Deploy(ctx, cdep) require.NoError(t, err) } diff --git a/cluster/kube/invendory_node.go b/cluster/kube/invendory_node.go new file mode 100644 index 000000000..80560e53a --- /dev/null +++ b/cluster/kube/invendory_node.go @@ -0,0 +1,200 @@ +package kube + +import ( + "fmt" + + types "github.com/akash-network/akash-api/go/node/types/v1beta3" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/akash-network/provider/cluster/kube/builder" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" + crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" +) + +const ( + runtimeClassNvidia = "nvidia" +) + +type node struct { + id string + cpu resourcePair + gpu resourcePair + memory resourcePair + ephemeralStorage resourcePair + volumesAttached resourcePair + volumesMounted resourcePair + capabilities *crd.NodeInfoCapabilities +} + +func newNode(nodeStatus *corev1.NodeStatus, capabilities *crd.NodeInfoCapabilities) *node { + mzero := resource.NewMilliQuantity(0, resource.DecimalSI) + zero := resource.NewQuantity(0, resource.DecimalSI) + + gpu := *resource.NewQuantity(0, resource.DecimalSI) + if capabilities != nil { + var resourceName corev1.ResourceName + switch capabilities.GPU.Vendor { + case builder.GPUVendorNvidia: + resourceName = builder.ResourceGPUNvidia + case builder.GPUVendorAMD: + resourceName = builder.ResourceGPUAMD + } + + gpu = nodeStatus.Allocatable.Name(resourceName, resource.DecimalSI).DeepCopy() + } + + nd := &node{ + cpu: newResourcePair(nodeStatus.Allocatable.Cpu().DeepCopy(), mzero.DeepCopy()), + gpu: newResourcePair(gpu, zero.DeepCopy()), + memory: newResourcePair(nodeStatus.Allocatable.Memory().DeepCopy(), zero.DeepCopy()), + ephemeralStorage: newResourcePair(nodeStatus.Allocatable.StorageEphemeral().DeepCopy(), zero.DeepCopy()), + volumesAttached: newResourcePair(*resource.NewQuantity(int64(len(nodeStatus.VolumesAttached)), resource.DecimalSI), zero.DeepCopy()), + capabilities: capabilities, + } + + return nd +} + +func (nd *node) addAllocatedResources(rl corev1.ResourceList) { + for name, quantity := range rl { + switch name { + case corev1.ResourceCPU: + nd.cpu.allocated.Add(quantity) + case corev1.ResourceMemory: + nd.memory.allocated.Add(quantity) + case corev1.ResourceEphemeralStorage: + nd.ephemeralStorage.allocated.Add(quantity) + case builder.ResourceGPUNvidia: + fallthrough + case builder.ResourceGPUAMD: + nd.gpu.allocated.Add(quantity) + } + } +} + +func (nd *node) dup() *node { + res := &node{ + id: nd.id, + cpu: *nd.cpu.dup(), + gpu: *nd.gpu.dup(), + memory: *nd.memory.dup(), + ephemeralStorage: *nd.ephemeralStorage.dup(), + volumesAttached: *nd.volumesAttached.dup(), + volumesMounted: *nd.volumesMounted.dup(), + capabilities: nd.capabilities.DeepCopy(), + } + + return res +} + +func (nd *node) tryAdjustCPU(res *types.CPU) bool { + return nd.cpu.subMilliNLZ(res.Units) +} + +func (nd *node) tryAdjustGPU(res *types.GPU, sparams *crd.SchedulerParams) bool { + if res.Units.Value() == 0 { + return true + } + + // GPUs cannot be reserved until node capabilities available + if nd.capabilities == nil { + return false + } + + attrs, err := ctypes.ParseGPUAttributes(res.Attributes) + if err != nil { + return false + } + + models, match := attrs[nd.capabilities.GPU.Vendor] + if !match { + return false + } + + var model string + for _, m := range models { + if m == nd.capabilities.GPU.Model || m == "*" { + model = nd.capabilities.GPU.Model + break + } + } + + if model == "" { + return false + } + + if !nd.gpu.subNLZ(res.Units) { + return false + } + + sParamsEnsureGPU(sparams) + sparams.Resources.GPU.Vendor = nd.capabilities.GPU.Vendor + sparams.Resources.GPU.Model = model + + switch nd.capabilities.GPU.Vendor { + case builder.GPUVendorNvidia: + sparams.RuntimeClass = runtimeClassNvidia + default: + } + + res.Attributes = types.Attributes{ + { + Key: fmt.Sprintf("vendor/%s/model/%s", nd.capabilities.GPU.Vendor, model), + Value: "true", + }, + } + + return true +} + +func sParamsEnsureGPU(sparams *crd.SchedulerParams) { + sParamsEnsureResources(sparams) + + if sparams.Resources.GPU == nil { + sparams.Resources.GPU = &crd.SchedulerResourceGPU{} + } +} + +func sParamsEnsureResources(sparams *crd.SchedulerParams) { + if sparams.Resources == nil { + sparams.Resources = &crd.SchedulerResources{} + } +} + +// +// func sParamsEnsureAffinityNode(sparams *crd.SchedulerParams) { +// sParamsEnsureAffinity(sparams) +// +// if sparams.Affinity.Node == nil { +// sparams.Affinity.Node = &crd.NodeAffinity{} +// } +// } +// +// func sParamsApplySelectors(sparams *crd.SchedulerParams, selectors []corev1.NodeSelectorRequirement) { +// sParamsEnsureAffinityNode(sparams) +// +// sparams.Affinity.Node.Required = append(sparams.Affinity.Node.Required, selectors...) +// } + +func (nd *node) tryAdjustMemory(res *types.Memory) bool { + return nd.memory.subNLZ(res.Quantity) +} + +func (nd *node) tryAdjustEphemeralStorage(res *types.Storage) bool { + return nd.ephemeralStorage.subNLZ(res.Quantity) +} + +// nolint: unused +func (nd *node) tryAdjustVolumesAttached(res types.ResourceValue) bool { + return nd.volumesAttached.subNLZ(res) +} + +func (cn clusterNodes) dup() clusterNodes { + ret := make(clusterNodes) + + for name, nd := range cn { + ret[name] = nd.dup() + } + return ret +} diff --git a/cluster/kube/inventory.go b/cluster/kube/inventory.go index 2907a9316..29f96ed19 100644 --- a/cluster/kube/inventory.go +++ b/cluster/kube/inventory.go @@ -3,19 +3,17 @@ package kube import ( "context" "fmt" + "reflect" "strings" "time" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/pager" types "github.com/akash-network/akash-api/go/node/types/v1beta3" - "github.com/akash-network/node/sdl" - metricsutils "github.com/akash-network/node/util/metrics" "github.com/akash-network/provider/cluster/kube/builder" ctypes "github.com/akash-network/provider/cluster/types/v1beta3" @@ -26,18 +24,6 @@ const ( inventoryOperatorQueryTimeout = 5 * time.Second ) -type node struct { - id string - arch string - cpu resourcePair - gpu resourcePair - memory resourcePair - ephemeralStorage resourcePair - volumesAttached resourcePair - volumesMounted resourcePair - storageClasses map[string]bool -} - type clusterNodes map[string]*node type inventory struct { @@ -65,123 +51,129 @@ func (inv *inventory) dup() inventory { return dup } -func (nd *node) allowsStorageClasses(volumes types.Volumes) bool { - for _, storage := range volumes { - attr := storage.Attributes.Find(sdl.StorageAttributePersistent) - if persistent, set := attr.AsBool(); !set || !persistent { - continue +// tryAdjust cluster inventory +// It returns two boolean values. First indicates if node-wide resources satisfy (true) requirements +// Seconds indicates if cluster-wide resources satisfy (true) requirements +func (inv *inventory) tryAdjust(node string, res *types.ResourceUnits) (*crd.SchedulerParams, bool, bool) { + nd := inv.nodes[node].dup() + sparams := &crd.SchedulerParams{} + + if !nd.tryAdjustCPU(res.CPU) { + return nil, false, true + } + + if !nd.tryAdjustGPU(res.GPU, sparams) { + return nil, false, true + } + + if !nd.tryAdjustMemory(res.Memory) { + return nil, false, true + } + + storageClasses := inv.storageClasses.dup() + + for i, storage := range res.Storage { + attrs, err := ctypes.ParseStorageAttributes(storage.Attributes) + if err != nil { + return nil, false, false } - attr = storage.Attributes.Find(sdl.StorageAttributeClass) - if class, set := attr.AsString(); set { - if _, allowed := nd.storageClasses[class]; !allowed { - return false + if !attrs.Persistent { + if !nd.tryAdjustEphemeralStorage(&res.Storage[i]) { + return nil, false, true } + continue + } + + if !nd.capabilities.Storage.HasClass(attrs.Class) { + return nil, false, true + } + + // if !nd.tryAdjustVolumesAttached(types.NewResourceValue(1)) { + // return nil, false, true + // } + + // no need to check if storageClass map has class present as it has been validated + // for particular node during inventory fetch + if !storageClasses[attrs.Class].subNLZ(storage.Quantity) { + // cluster storage does not have enough space thus break to error + return nil, false, false } } - return true + // all requirements for current group have been satisfied + // commit and move on + inv.nodes[node] = nd + inv.storageClasses = storageClasses + + if reflect.DeepEqual(sparams, &crd.SchedulerParams{}) { + return nil, true, true + } + + return sparams, true, true } -func (inv *inventory) Adjust(reservation ctypes.Reservation) error { +func (inv *inventory) Adjust(reservation ctypes.ReservationGroup, opts ...ctypes.InventoryOption) error { + cfg := &ctypes.InventoryOptions{} + for _, opt := range opts { + cfg = opt(cfg) + } + resources := make([]types.Resources, len(reservation.Resources().GetResources())) + adjustedResources := make([]types.Resources, 0, len(reservation.Resources().GetResources())) copy(resources, reservation.Resources().GetResources()) + cparams := crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(reservation.Resources().GetResources())), + } + currInventory := inv.dup() nodes: for nodeName := range currInventory.nodes { for i := len(resources) - 1; i >= 0; i-- { - res := resources[i].Resources - for ; resources[i].Count > 0; resources[i].Count-- { - nd := currInventory.nodes[nodeName] + adjusted := resources[i] - // first check if reservation needs persistent storage - // and node handles such class - if !nd.allowsStorageClasses(res.Storage) { - continue nodes - } - - var adjusted bool - - cpu := nd.cpu.dup() - if adjusted = cpu.subMilliNLZ(res.CPU.Units); !adjusted { - continue nodes - } - - gpu := nd.gpu.dup() - if res.GPU != nil { - if adjusted = gpu.subNLZ(res.GPU.Units); !adjusted { - continue nodes - } + for ; resources[i].Count > 0; resources[i].Count-- { + sparams, nStatus, cStatus := currInventory.tryAdjust(nodeName, &adjusted.Resources) + if !cStatus { + // cannot satisfy cluster-wide resources, stop lookup + break nodes } - memory := nd.memory.dup() - if adjusted = memory.subNLZ(res.Memory.Quantity); !adjusted { + if !nStatus { + // cannot satisfy node-wide resources, try with next node continue nodes } - ephemeralStorage := nd.ephemeralStorage.dup() - volumesAttached := nd.volumesAttached.dup() - - storageClasses := currInventory.storageClasses.dup() - - for _, storage := range res.Storage { - attr := storage.Attributes.Find(sdl.StorageAttributePersistent) - - if persistent, _ := attr.AsBool(); !persistent { - if adjusted = ephemeralStorage.subNLZ(storage.Quantity); !adjusted { - continue nodes - } - continue - } - - // if volumesAttached, adjusted = volumesAttached.subNLZ(types.NewResourceValue(1)); !adjusted { - // continue nodes - // } - - attr = storage.Attributes.Find(sdl.StorageAttributeClass) - class, _ := attr.AsString() - - cstorage, isAvailable := storageClasses[class] - if !isAvailable { - break nodes - } - - if adjusted = cstorage.subNLZ(storage.Quantity); !adjusted { - // cluster storage does not have enough space thus break to error + if sparams != nil { + if cparams.SchedulerParams[i] == nil { + cparams.SchedulerParams[i] = sparams + } else if !reflect.DeepEqual(sparams, cparams.SchedulerParams[i]) { + // all replicas of the same service are expected to have same node selectors and runtimes + // if they don't match then provider cannot bid break nodes } } - - // all requirements for current group have been satisfied - // commit and move on - currInventory.nodes[nodeName] = &node{ - id: nd.id, - arch: nd.arch, - cpu: *cpu, - gpu: *gpu, - memory: *memory, - ephemeralStorage: *ephemeralStorage, - volumesAttached: *volumesAttached, - volumesMounted: nd.volumesMounted, - storageClasses: nd.storageClasses, - } - - currInventory.storageClasses = storageClasses } // all replicas resources are fulfilled when count == 0. // remove group from the list to prevent double request of the same resources if resources[i].Count == 0 { - // tmpResources = append(tmpResources, resources[rIdx]) resources = append(resources[:i], resources[i+1:]...) + adjustedResources = append(adjustedResources, adjusted) } } } if len(resources) == 0 { - *inv = currInventory + if !cfg.DryRun { + *inv = currInventory + } + + reservation.SetAllocatedResources(adjustedResources) + reservation.SetClusterParams(cparams) + return nil } @@ -289,7 +281,6 @@ func (c *client) fetchStorage(ctx context.Context) (clusterStorage, error) { cstorage := make(clusterStorage) - // TODO - figure out if we can get this back via ServiceDiscoveryAgent query akash-services inventory-operator api // discover inventory operator // empty namespace mean search through all namespaces svcResult, err := c.kc.CoreV1().Services(corev1.NamespaceAll).List(ctx, metav1.ListOptions{ @@ -345,14 +336,64 @@ func (c *client) fetchStorage(ctx context.Context) (clusterStorage, error) { return cstorage, nil } +// todo write unmarshaler +func parseNodeCapabilities(labels map[string]string, cStorage clusterStorage) *crd.NodeInfoCapabilities { + capabilities := &crd.NodeInfoCapabilities{} + + for k := range labels { + tokens := strings.Split(k, "/") + if len(tokens) < 3 || + tokens[0] != builder.AkashManagedLabelName || + tokens[1] != "capability" { + continue + } + + tokens = tokens[2:] + switch tokens[0] { + case "gpu": + if len(tokens) < 1 { + continue + } + + tokens = tokens[1:] + if tokens[0] == "vendor" { + capabilities.GPU.Vendor = tokens[1] + if tokens[2] == "model" { + capabilities.GPU.Vendor = tokens[3] + } + } + case "storage": + if len(tokens) < 2 { + continue + } + + switch tokens[1] { + case "class": + capabilities.Storage.Classes = append(capabilities.Storage.Classes, tokens[2]) + default: + } + } + } + + // parse storage classes with legacy mode if new mode is not detected + if len(capabilities.Storage.Classes) == 0 { + if value, defined := labels[builder.AkashNetworkStorageClasses]; defined { + for _, class := range strings.Split(value, ".") { + if _, avail := cStorage[class]; avail { + capabilities.Storage.Classes = append(capabilities.Storage.Classes, class) + } + } + } + } + + return capabilities +} + func (c *client) fetchActiveNodes(ctx context.Context, cstorage clusterStorage) (map[string]*node, error) { // todo filter nodes by akash.network label - knodes, err := c.kc.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - label := metricsutils.SuccessLabel - if err != nil { - label = metricsutils.FailLabel - } - kubeCallsCounter.WithLabelValues("nodes-list", label).Inc() + knodes, err := wrapKubeCall("nodes-list", func() (*corev1.NodeList, error) { + return c.kc.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + }) if err != nil { return nil, err } @@ -364,7 +405,6 @@ func (c *client) fetchActiveNodes(ctx context.Context, cstorage clusterStorage) podsPager := pager.New(func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { return podsClient.List(ctx, opts) }) - zero := resource.NewMilliQuantity(0, "m") retnodes := make(map[string]*node) for _, knode := range knodes.Items { @@ -372,46 +412,9 @@ func (c *client) fetchActiveNodes(ctx context.Context, cstorage clusterStorage) continue } - // Create an entry with the allocatable amount for the node - cpu := knode.Status.Allocatable.Cpu().DeepCopy() - gpu := knode.Status.Allocatable.Name(builder.ResourceNvidiaGPU, resource.DecimalSI).DeepCopy() - memory := knode.Status.Allocatable.Memory().DeepCopy() - storage := knode.Status.Allocatable.StorageEphemeral().DeepCopy() - entry := &node{ - arch: knode.Status.NodeInfo.Architecture, - cpu: resourcePair{ - allocatable: cpu, - }, - gpu: resourcePair{ - allocatable: gpu, - }, - memory: resourcePair{ - allocatable: memory, - }, - ephemeralStorage: resourcePair{ - allocatable: storage, - }, - volumesAttached: resourcePair{ - allocated: *resource.NewQuantity(int64(len(knode.Status.VolumesAttached)), resource.DecimalSI), - }, - storageClasses: make(map[string]bool), - } + capabilities := parseNodeCapabilities(knode.Labels, cstorage) - if value, defined := knode.Labels[builder.AkashNetworkStorageClasses]; defined { - for _, class := range strings.Split(value, ".") { - if _, avail := cstorage[class]; avail { - entry.storageClasses[class] = true - } - } - } - - // Initialize the allocated amount to for each node - zero.DeepCopyInto(&entry.cpu.allocated) - zero.DeepCopyInto(&entry.gpu.allocated) - zero.DeepCopyInto(&entry.memory.allocated) - zero.DeepCopyInto(&entry.ephemeralStorage.allocated) - - retnodes[knode.Name] = entry + retnodes[knode.Name] = newNode(&knode.Status, capabilities) } // Go over each pod and sum the resources for it into the value for the pod it lives on @@ -433,6 +436,7 @@ func (c *client) fetchActiveNodes(ctx context.Context, cstorage clusterStorage) entry.addAllocatedResources(pod.Spec.Overhead) retnodes[nodeName] = entry // Map is by value, so store the copy back into the map + return nil }) @@ -443,49 +447,6 @@ func (c *client) fetchActiveNodes(ctx context.Context, cstorage clusterStorage) return retnodes, nil } -func (nd *node) addAllocatedResources(rl corev1.ResourceList) { - for name, quantity := range rl { - switch name { - case corev1.ResourceCPU: - nd.cpu.allocated.Add(quantity) - case corev1.ResourceMemory: - nd.memory.allocated.Add(quantity) - case corev1.ResourceEphemeralStorage: - nd.ephemeralStorage.allocated.Add(quantity) - case builder.ResourceNvidiaGPU: - nd.gpu.allocated.Add(quantity) - } - } -} - -func (nd *node) dup() *node { - res := &node{ - id: nd.id, - arch: nd.arch, - cpu: *nd.cpu.dup(), - gpu: *nd.gpu.dup(), - memory: *nd.memory.dup(), - ephemeralStorage: *nd.ephemeralStorage.dup(), - volumesAttached: *nd.volumesAttached.dup(), - volumesMounted: *nd.volumesMounted.dup(), - storageClasses: make(map[string]bool), - } - - for k, v := range nd.storageClasses { - res.storageClasses[k] = v - } - - return res -} - -func (cn clusterNodes) dup() clusterNodes { - ret := make(clusterNodes) - - for name, nd := range cn { - ret[name] = nd.dup() - } - return ret -} func (c *client) nodeIsActive(node corev1.Node) bool { ready := false issues := 0 diff --git a/cluster/kube/inventory_test.go b/cluster/kube/inventory_test.go index e8455f754..3730b01a8 100644 --- a/cluster/kube/inventory_test.go +++ b/cluster/kube/inventory_test.go @@ -26,7 +26,8 @@ import ( ) type testReservation struct { - resources dtypes.GroupSpec + resources dtypes.GroupSpec + adjustedResources []atypes.Resources } var _ ctypes.Reservation = (*testReservation)(nil) @@ -39,10 +40,17 @@ func (r *testReservation) Resources() atypes.ResourceGroup { return r.resources } +func (r *testReservation) SetAllocatedResources(val []atypes.Resources) { + r.adjustedResources = val +} + func (r *testReservation) Allocated() bool { return false } +func (r *testReservation) SetClusterParams(_ interface{}) {} +func (r *testReservation) ClusterParams() interface{} { return nil } + type inventoryScaffold struct { kmock *kubernetesmocks.Interface amock *akashclientfake.Clientset diff --git a/cluster/kube/k8s_integration_test.go b/cluster/kube/k8s_integration_test.go index 85d31eb92..725b892ab 100644 --- a/cluster/kube/k8s_integration_test.go +++ b/cluster/kube/k8s_integration_test.go @@ -20,7 +20,9 @@ import ( "github.com/akash-network/provider/cluster/kube/builder" "github.com/akash-network/provider/cluster/kube/clientcommon" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" providerflags "github.com/akash-network/provider/cmd/provider-services/cmd/flags" + crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" mtestutil "github.com/akash-network/provider/testutil/manifest/v2beta2" ) @@ -103,8 +105,16 @@ func TestNewClient(t *testing.T) { assert.NoError(t, err) require.Empty(t, deployments) + cdep := &ctypes.Deployment{ + Lid: lid, + MGroup: &group, + CParams: crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(group.Services)), + }, + } + // deploy lease - err = ac.Deploy(ctx, lid, &group) + err = ac.Deploy(ctx, cdep) assert.NoError(t, err) // query deployments, ensure lease present diff --git a/cluster/kube/resourcetypes.go b/cluster/kube/resourcetypes.go index 806159914..773b71e5d 100644 --- a/cluster/kube/resourcetypes.go +++ b/cluster/kube/resourcetypes.go @@ -19,7 +19,7 @@ type resourcePair struct { type clusterStorage map[string]*resourcePair func (cs clusterStorage) dup() clusterStorage { - res := make(clusterStorage, len(cs)) + res := make(clusterStorage) for class, resources := range cs { res[class] = resources.dup() } @@ -27,6 +27,15 @@ func (cs clusterStorage) dup() clusterStorage { return res } +func newResourcePair(allocatable, allocated resource.Quantity) resourcePair { + rp := resourcePair{ + allocatable: allocatable, + allocated: allocated, + } + + return rp +} + func rpNewFromAkash(res crd.ResourcePair) *resourcePair { return &resourcePair{ allocatable: *resource.NewQuantity(int64(res.Allocatable), resource.DecimalSI), @@ -52,6 +61,7 @@ func (rp *resourcePair) subMilliNLZ(val types.ResourceValue) bool { allocated := rp.allocated.DeepCopy() allocated.Add(*resource.NewMilliQuantity(int64(val.Value()), resource.DecimalSI)) + *rp = resourcePair{ allocatable: rp.allocatable.DeepCopy(), allocated: allocated, @@ -81,7 +91,7 @@ func (rp *resourcePair) subNLZ(val types.ResourceValue) bool { return true } -func (rp resourcePair) available() resource.Quantity { +func (rp *resourcePair) available() resource.Quantity { result := rp.allocatable.DeepCopy() if result.Value() == -1 { diff --git a/cluster/manager.go b/cluster/manager.go index 9f297d34c..92a78385b 100644 --- a/cluster/manager.go +++ b/cluster/manager.go @@ -54,42 +54,37 @@ var ( ) type deploymentManager struct { - bus pubsub.Bus - client Client - session session.Session - - state deploymentState - - lease mtypes.LeaseID - mgroup *mani.Group - - monitor *deploymentMonitor - wg sync.WaitGroup - updatech chan *mani.Group - teardownch chan struct{} - currentHostnames map[string]struct{} - - log log.Logger - lc lifecycle.Lifecycle - hostnameService ctypes.HostnameServiceClient - - config Config - + bus pubsub.Bus + client Client + session session.Session + state deploymentState + deployment ctypes.IDeployment + monitor *deploymentMonitor + wg sync.WaitGroup + updatech chan ctypes.IDeployment + teardownch chan struct{} + currentHostnames map[string]struct{} + log log.Logger + lc lifecycle.Lifecycle + hostnameService ctypes.HostnameServiceClient + config Config serviceShuttingDown <-chan struct{} } -func newDeploymentManager(s *service, lease mtypes.LeaseID, mgroup *mani.Group, isNewLease bool) *deploymentManager { - logger := s.log.With("cmp", "deployment-manager", "lease", lease, "manifest-group", mgroup.GetName()) +func newDeploymentManager(s *service, deployment ctypes.IDeployment, isNewLease bool) *deploymentManager { + lid := deployment.LeaseID() + mgroup := deployment.ManifestGroup() + + logger := s.log.With("cmp", "deployment-manager", "lease", lid, "manifest-group", mgroup.GetName()) dm := &deploymentManager{ bus: s.bus, client: s.client, session: s.session, state: dsDeployActive, - lease: lease, - mgroup: mgroup, + deployment: deployment, wg: sync.WaitGroup{}, - updatech: make(chan *mani.Group), + updatech: make(chan ctypes.IDeployment), teardownch: make(chan struct{}), log: logger, lc: lifecycle.New(), @@ -108,17 +103,17 @@ func newDeploymentManager(s *service, lease mtypes.LeaseID, mgroup *mani.Group, s.managerch <- dm }() - err := s.bus.Publish(event.LeaseAddFundsMonitor{LeaseID: lease, IsNewLease: isNewLease}) + err := s.bus.Publish(event.LeaseAddFundsMonitor{LeaseID: lid, IsNewLease: isNewLease}) if err != nil { - s.log.Error("unable to publish LeaseAddFundsMonitor event", "error", err, "lease", lease) + s.log.Error("unable to publish LeaseAddFundsMonitor event", "error", err, "lease", lid) } return dm } -func (dm *deploymentManager) update(mgroup *mani.Group) error { +func (dm *deploymentManager) update(deployment ctypes.IDeployment) error { select { - case dm.updatech <- mgroup: + case dm.updatech <- deployment: return nil case <-dm.lc.ShuttingDown(): return ErrNotRunning @@ -155,7 +150,7 @@ func (dm *deploymentManager) run(ctx context.Context) { runch := dm.startDeploy(ctx) defer func() { - err := dm.hostnameService.ReleaseHostnames(dm.lease) + err := dm.hostnameService.ReleaseHostnames(dm.deployment.LeaseID()) if err != nil { dm.log.Error("failed releasing hostnames", "err", err) } @@ -170,8 +165,8 @@ loop: case shutdownErr = <-dm.lc.ShutdownRequest(): break loop - case mgroup := <-dm.updatech: - dm.mgroup = mgroup + case deployment := <-dm.updatech: + dm.deployment = deployment newch := dm.handleUpdate(ctx) if newch != nil { runch = newch @@ -188,7 +183,6 @@ loop: // Run the teardown code to get rid of anything created that might be hanging out runch = dm.startTeardown() } else { - dm.log.Debug("deploy complete") dm.state = dsDeployComplete dm.startMonitor() @@ -239,7 +233,7 @@ loop: dm.log.Debug("waiting on dm.wg") dm.wg.Wait() - if dm.state != dsDeployComplete { + if dm.state < dsDeployComplete { dm.log.Info("shutting down unclean, running teardown now") ctx, cancel := context.WithTimeout(context.Background(), uncleanShutdownGracePeriod) defer cancel() @@ -285,17 +279,17 @@ func (dm *deploymentManager) startDeploy(ctx context.Context) <-chan error { if len(hostnames) != 0 { // Some hostnames have been withheld - dm.log.Info("hostnames withheld from deployment", "cnt", len(hostnames), "lease", dm.lease) + dm.log.Info("hostnames withheld from deployment", "cnt", len(hostnames), "lease", dm.deployment.LeaseID()) } if len(endpoints) != 0 { // Some endpoints have been withheld - dm.log.Info("endpoints withheld from deployment", "cnt", len(endpoints), "lease", dm.lease) + dm.log.Info("endpoints withheld from deployment", "cnt", len(endpoints), "lease", dm.deployment.LeaseID()) } - groupCopy := *dm.mgroup + groupCopy := *dm.deployment.ManifestGroup() ev := event.ClusterDeployment{ - LeaseID: dm.lease, + LeaseID: dm.deployment.LeaseID(), Group: &groupCopy, Status: event.ClusterDeploymentUpdated, } @@ -324,7 +318,7 @@ type serviceExposeWithServiceName struct { } func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, error) { - cleanupHelper := newDeployCleanupHelper(dm.lease, dm.client, dm.log) + cleanupHelper := newDeployCleanupHelper(dm.deployment.LeaseID(), dm.client, dm.log) var err error ctx, cancel := context.WithCancel(context.Background()) @@ -348,14 +342,14 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, return nil, nil, err } - currentIPs, err := dm.client.GetDeclaredIPs(ctx, dm.lease) + currentIPs, err := dm.client.GetDeclaredIPs(ctx, dm.deployment.LeaseID()) if err != nil { return nil, nil, err } // Either reserve the hostnames, or confirm that they already are held - allHostnames := manifest.AllHostnamesOfManifestGroup(*dm.mgroup) - withheldHostnames, err := dm.hostnameService.ReserveHostnames(ctx, allHostnames, dm.lease) + allHostnames := manifest.AllHostnamesOfManifestGroup(*dm.deployment.ManifestGroup()) + withheldHostnames, err := dm.hostnameService.ReserveHostnames(ctx, allHostnames, dm.deployment.LeaseID()) if err != nil { deploymentCounter.WithLabelValues("reserve-hostnames", "err").Inc() @@ -382,12 +376,16 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, // Don't use a context tied to the lifecycle, as we don't want to cancel Kubernetes operations deployCtx := util.ApplyToContext(context.Background(), dm.config.ClusterSettings) - err = dm.client.Deploy(deployCtx, dm.lease, dm.mgroup) + err = dm.client.Deploy(deployCtx, dm.deployment) label := "success" if err != nil { label = "fail" } deploymentCounter.WithLabelValues("deploy", label).Inc() + if err != nil { + dm.log.Error("deploying workload", "err", err.Error()) + return nil, nil, err + } // Figure out what hostnames to declare blockedHostnames := make(map[string]struct{}) @@ -402,11 +400,11 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, // clear this out so it gets repopulated dm.currentHostnames = make(map[string]struct{}) // Iterate over each entry, extracting the ingress services & leased IPs - for _, service := range dm.mgroup.Services { + for _, service := range dm.deployment.ManifestGroup().Services { for _, expose := range service.Expose { if sdlutil.ShouldBeIngress(expose) { if dm.config.DeploymentIngressStaticHosts { - uid := manifest.IngressHost(dm.lease, service.Name) + uid := manifest.IngressHost(dm.deployment.LeaseID(), service.Name) host := fmt.Sprintf("%s.%s", uid, dm.config.DeploymentIngressDomain) hosts[host] = expose hostToServiceName[host] = service.Name @@ -425,7 +423,7 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, if expose.Global && len(expose.IP) != 0 { v := serviceExposeWithServiceName{expose: expose, name: service.Name} leasedIPs = append(leasedIPs, v) - sharingKey := clusterutil.MakeIPSharingKey(dm.lease, expose.IP) + sharingKey := clusterutil.MakeIPSharingKey(dm.deployment.LeaseID(), expose.IP) ipsInThisRequest[sharingKey] = v } } @@ -445,7 +443,7 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, for host, serviceExpose := range hosts { externalPort := uint32(sdlutil.ExposeExternalPort(serviceExpose)) - err = dm.client.DeclareHostname(ctx, dm.lease, host, hostToServiceName[host], externalPort) + err = dm.client.DeclareHostname(ctx, dm.deployment.LeaseID(), host, hostToServiceName[host], externalPort) if err != nil { // TODO - counter return withheldHostnames, nil, err @@ -455,12 +453,12 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, withheldEndpoints := make([]string, 0) for _, serviceExpose := range leasedIPs { endpointName := serviceExpose.expose.IP - sharingKey := clusterutil.MakeIPSharingKey(dm.lease, endpointName) + sharingKey := clusterutil.MakeIPSharingKey(dm.deployment.LeaseID(), endpointName) externalPort := sdlutil.ExposeExternalPort(serviceExpose.expose) port := serviceExpose.expose.Port - err = dm.client.DeclareIP(ctx, dm.lease, serviceExpose.name, uint32(port), uint32(externalPort), serviceExpose.expose.Proto, sharingKey, false) + err = dm.client.DeclareIP(ctx, dm.deployment.LeaseID(), serviceExpose.name, uint32(port), uint32(externalPort), serviceExpose.expose.Proto, sharingKey, false) if err != nil { if !errors.Is(err, kubeclienterrors.ErrAlreadyExists) { dm.log.Error("failed adding IP declaration", "service", serviceExpose.name, "port", externalPort, "endpoint", serviceExpose.expose.IP, "err", err) @@ -480,8 +478,8 @@ func (dm *deploymentManager) doDeploy(ctx context.Context) ([]string, []string, func (dm *deploymentManager) getCleanupRetryOpts(ctx context.Context) []retry.Option { retryFn := func(err error) bool { isCanceled := errors.Is(err, context.Canceled) - isDeadlineExceeeded := errors.Is(err, context.DeadlineExceeded) - return !isCanceled && !isDeadlineExceeeded + isDeadlineExceeded := errors.Is(err, context.DeadlineExceeded) + return !isCanceled && !isDeadlineExceeded } return []retry.Option{ retry.Attempts(50), @@ -500,7 +498,7 @@ func (dm *deploymentManager) doTeardown(ctx context.Context) error { go func() { result := retry.Do(func() error { - err := dm.client.TeardownLease(ctx, dm.lease) + err := dm.client.TeardownLease(ctx, dm.deployment.LeaseID()) if err != nil { dm.log.Error("lease teardown failed", "err", err) } @@ -517,7 +515,7 @@ func (dm *deploymentManager) doTeardown(ctx context.Context) error { go func() { result := retry.Do(func() error { - err := dm.client.PurgeDeclaredHostnames(ctx, dm.lease) + err := dm.client.PurgeDeclaredHostnames(ctx, dm.deployment.LeaseID()) if err != nil { dm.log.Error("purge declared hostname failure", "err", err) } @@ -533,7 +531,7 @@ func (dm *deploymentManager) doTeardown(ctx context.Context) error { go func() { result := retry.Do(func() error { - err := dm.client.PurgeDeclaredIPs(ctx, dm.lease) + err := dm.client.PurgeDeclaredIPs(ctx, dm.deployment.LeaseID()) if err != nil { dm.log.Error("purge declared ips failure", "err", err) } @@ -563,13 +561,12 @@ func (dm *deploymentManager) doTeardown(ctx context.Context) error { } func (dm *deploymentManager) checkLeaseActive(ctx context.Context) error { - var lease *mtypes.QueryLeaseResponse err := retry.Do(func() error { var err error lease, err = dm.session.Client().Query().Lease(ctx, &mtypes.QueryLeaseRequest{ - ID: dm.lease, + ID: dm.deployment.LeaseID(), }) if err != nil { dm.log.Error("lease query failed", "err") @@ -588,7 +585,7 @@ func (dm *deploymentManager) checkLeaseActive(ctx context.Context) error { if lease.GetLease().State != mtypes.LeaseActive { dm.log.Error("lease not active, not deploying") - return fmt.Errorf("%w: %s", ErrLeaseInactive, dm.lease) + return fmt.Errorf("%w: %s", ErrLeaseInactive, dm.deployment.LeaseID()) } return nil diff --git a/cluster/mocks/client.go b/cluster/mocks/client.go index 784da12cf..8f097c891 100644 --- a/cluster/mocks/client.go +++ b/cluster/mocks/client.go @@ -227,13 +227,13 @@ func (_c *Client_DeclareIP_Call) RunAndReturn(run func(context.Context, marketv1 return _c } -// Deploy provides a mock function with given fields: ctx, lID, mgroup -func (_m *Client) Deploy(ctx context.Context, lID marketv1beta3.LeaseID, mgroup *v2beta2.Group) error { - ret := _m.Called(ctx, lID, mgroup) +// Deploy provides a mock function with given fields: ctx, deployment +func (_m *Client) Deploy(ctx context.Context, deployment v1beta3.IDeployment) error { + ret := _m.Called(ctx, deployment) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, marketv1beta3.LeaseID, *v2beta2.Group) error); ok { - r0 = rf(ctx, lID, mgroup) + if rf, ok := ret.Get(0).(func(context.Context, v1beta3.IDeployment) error); ok { + r0 = rf(ctx, deployment) } else { r0 = ret.Error(0) } @@ -248,15 +248,14 @@ type Client_Deploy_Call struct { // Deploy is a helper method to define mock.On call // - ctx context.Context -// - lID marketv1beta3.LeaseID -// - mgroup *v2beta2.Group -func (_e *Client_Expecter) Deploy(ctx interface{}, lID interface{}, mgroup interface{}) *Client_Deploy_Call { - return &Client_Deploy_Call{Call: _e.mock.On("Deploy", ctx, lID, mgroup)} +// - deployment v1beta3.IDeployment +func (_e *Client_Expecter) Deploy(ctx interface{}, deployment interface{}) *Client_Deploy_Call { + return &Client_Deploy_Call{Call: _e.mock.On("Deploy", ctx, deployment)} } -func (_c *Client_Deploy_Call) Run(run func(ctx context.Context, lID marketv1beta3.LeaseID, mgroup *v2beta2.Group)) *Client_Deploy_Call { +func (_c *Client_Deploy_Call) Run(run func(ctx context.Context, deployment v1beta3.IDeployment)) *Client_Deploy_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(marketv1beta3.LeaseID), args[2].(*v2beta2.Group)) + run(args[0].(context.Context), args[1].(v1beta3.IDeployment)) }) return _c } @@ -266,25 +265,25 @@ func (_c *Client_Deploy_Call) Return(_a0 error) *Client_Deploy_Call { return _c } -func (_c *Client_Deploy_Call) RunAndReturn(run func(context.Context, marketv1beta3.LeaseID, *v2beta2.Group) error) *Client_Deploy_Call { +func (_c *Client_Deploy_Call) RunAndReturn(run func(context.Context, v1beta3.IDeployment) error) *Client_Deploy_Call { _c.Call.Return(run) return _c } // Deployments provides a mock function with given fields: _a0 -func (_m *Client) Deployments(_a0 context.Context) ([]v1beta3.Deployment, error) { +func (_m *Client) Deployments(_a0 context.Context) ([]v1beta3.IDeployment, error) { ret := _m.Called(_a0) - var r0 []v1beta3.Deployment + var r0 []v1beta3.IDeployment var r1 error - if rf, ok := ret.Get(0).(func(context.Context) ([]v1beta3.Deployment, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context) ([]v1beta3.IDeployment, error)); ok { return rf(_a0) } - if rf, ok := ret.Get(0).(func(context.Context) []v1beta3.Deployment); ok { + if rf, ok := ret.Get(0).(func(context.Context) []v1beta3.IDeployment); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]v1beta3.Deployment) + r0 = ret.Get(0).([]v1beta3.IDeployment) } } @@ -315,12 +314,12 @@ func (_c *Client_Deployments_Call) Run(run func(_a0 context.Context)) *Client_De return _c } -func (_c *Client_Deployments_Call) Return(_a0 []v1beta3.Deployment, _a1 error) *Client_Deployments_Call { +func (_c *Client_Deployments_Call) Return(_a0 []v1beta3.IDeployment, _a1 error) *Client_Deployments_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *Client_Deployments_Call) RunAndReturn(run func(context.Context) ([]v1beta3.Deployment, error)) *Client_Deployments_Call { +func (_c *Client_Deployments_Call) RunAndReturn(run func(context.Context) ([]v1beta3.IDeployment, error)) *Client_Deployments_Call { _c.Call.Return(run) return _c } diff --git a/cluster/mocks/deployment.go b/cluster/mocks/deployment.go index 10ca33433..cbf26098a 100644 --- a/cluster/mocks/deployment.go +++ b/cluster/mocks/deployment.go @@ -3,10 +3,9 @@ package mocks import ( + marketv1beta3 "github.com/akash-network/akash-api/go/node/market/v1beta3" mock "github.com/stretchr/testify/mock" - v1beta3 "github.com/akash-network/akash-api/go/node/market/v1beta3" - v2beta2 "github.com/akash-network/akash-api/go/manifest/v2beta2" ) @@ -23,15 +22,58 @@ func (_m *Deployment) EXPECT() *Deployment_Expecter { return &Deployment_Expecter{mock: &_m.Mock} } +// ClusterParams provides a mock function with given fields: +func (_m *Deployment) ClusterParams() interface{} { + ret := _m.Called() + + var r0 interface{} + if rf, ok := ret.Get(0).(func() interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// Deployment_ClusterParams_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClusterParams' +type Deployment_ClusterParams_Call struct { + *mock.Call +} + +// ClusterParams is a helper method to define mock.On call +func (_e *Deployment_Expecter) ClusterParams() *Deployment_ClusterParams_Call { + return &Deployment_ClusterParams_Call{Call: _e.mock.On("ClusterParams")} +} + +func (_c *Deployment_ClusterParams_Call) Run(run func()) *Deployment_ClusterParams_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Deployment_ClusterParams_Call) Return(_a0 interface{}) *Deployment_ClusterParams_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Deployment_ClusterParams_Call) RunAndReturn(run func() interface{}) *Deployment_ClusterParams_Call { + _c.Call.Return(run) + return _c +} + // LeaseID provides a mock function with given fields: -func (_m *Deployment) LeaseID() v1beta3.LeaseID { +func (_m *Deployment) LeaseID() marketv1beta3.LeaseID { ret := _m.Called() - var r0 v1beta3.LeaseID - if rf, ok := ret.Get(0).(func() v1beta3.LeaseID); ok { + var r0 marketv1beta3.LeaseID + if rf, ok := ret.Get(0).(func() marketv1beta3.LeaseID); ok { r0 = rf() } else { - r0 = ret.Get(0).(v1beta3.LeaseID) + r0 = ret.Get(0).(marketv1beta3.LeaseID) } return r0 @@ -54,25 +96,27 @@ func (_c *Deployment_LeaseID_Call) Run(run func()) *Deployment_LeaseID_Call { return _c } -func (_c *Deployment_LeaseID_Call) Return(_a0 v1beta3.LeaseID) *Deployment_LeaseID_Call { +func (_c *Deployment_LeaseID_Call) Return(_a0 marketv1beta3.LeaseID) *Deployment_LeaseID_Call { _c.Call.Return(_a0) return _c } -func (_c *Deployment_LeaseID_Call) RunAndReturn(run func() v1beta3.LeaseID) *Deployment_LeaseID_Call { +func (_c *Deployment_LeaseID_Call) RunAndReturn(run func() marketv1beta3.LeaseID) *Deployment_LeaseID_Call { _c.Call.Return(run) return _c } // ManifestGroup provides a mock function with given fields: -func (_m *Deployment) ManifestGroup() v2beta2.Group { +func (_m *Deployment) ManifestGroup() *v2beta2.Group { ret := _m.Called() - var r0 v2beta2.Group - if rf, ok := ret.Get(0).(func() v2beta2.Group); ok { + var r0 *v2beta2.Group + if rf, ok := ret.Get(0).(func() *v2beta2.Group); ok { r0 = rf() } else { - r0 = ret.Get(0).(v2beta2.Group) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v2beta2.Group) + } } return r0 @@ -95,12 +139,12 @@ func (_c *Deployment_ManifestGroup_Call) Run(run func()) *Deployment_ManifestGro return _c } -func (_c *Deployment_ManifestGroup_Call) Return(_a0 v2beta2.Group) *Deployment_ManifestGroup_Call { +func (_c *Deployment_ManifestGroup_Call) Return(_a0 *v2beta2.Group) *Deployment_ManifestGroup_Call { _c.Call.Return(_a0) return _c } -func (_c *Deployment_ManifestGroup_Call) RunAndReturn(run func() v2beta2.Group) *Deployment_ManifestGroup_Call { +func (_c *Deployment_ManifestGroup_Call) RunAndReturn(run func() *v2beta2.Group) *Deployment_ManifestGroup_Call { _c.Call.Return(run) return _c } diff --git a/cluster/mocks/i_deployment.go b/cluster/mocks/i_deployment.go new file mode 100644 index 000000000..b232b59eb --- /dev/null +++ b/cluster/mocks/i_deployment.go @@ -0,0 +1,165 @@ +// Code generated by mockery v2.24.0. DO NOT EDIT. + +package mocks + +import ( + marketv1beta3 "github.com/akash-network/akash-api/go/node/market/v1beta3" + mock "github.com/stretchr/testify/mock" + + v2beta2 "github.com/akash-network/akash-api/go/manifest/v2beta2" +) + +// IDeployment is an autogenerated mock type for the IDeployment type +type IDeployment struct { + mock.Mock +} + +type IDeployment_Expecter struct { + mock *mock.Mock +} + +func (_m *IDeployment) EXPECT() *IDeployment_Expecter { + return &IDeployment_Expecter{mock: &_m.Mock} +} + +// ClusterParams provides a mock function with given fields: +func (_m *IDeployment) ClusterParams() interface{} { + ret := _m.Called() + + var r0 interface{} + if rf, ok := ret.Get(0).(func() interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// IDeployment_ClusterParams_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClusterParams' +type IDeployment_ClusterParams_Call struct { + *mock.Call +} + +// ClusterParams is a helper method to define mock.On call +func (_e *IDeployment_Expecter) ClusterParams() *IDeployment_ClusterParams_Call { + return &IDeployment_ClusterParams_Call{Call: _e.mock.On("ClusterParams")} +} + +func (_c *IDeployment_ClusterParams_Call) Run(run func()) *IDeployment_ClusterParams_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *IDeployment_ClusterParams_Call) Return(_a0 interface{}) *IDeployment_ClusterParams_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *IDeployment_ClusterParams_Call) RunAndReturn(run func() interface{}) *IDeployment_ClusterParams_Call { + _c.Call.Return(run) + return _c +} + +// LeaseID provides a mock function with given fields: +func (_m *IDeployment) LeaseID() marketv1beta3.LeaseID { + ret := _m.Called() + + var r0 marketv1beta3.LeaseID + if rf, ok := ret.Get(0).(func() marketv1beta3.LeaseID); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(marketv1beta3.LeaseID) + } + + return r0 +} + +// IDeployment_LeaseID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LeaseID' +type IDeployment_LeaseID_Call struct { + *mock.Call +} + +// LeaseID is a helper method to define mock.On call +func (_e *IDeployment_Expecter) LeaseID() *IDeployment_LeaseID_Call { + return &IDeployment_LeaseID_Call{Call: _e.mock.On("LeaseID")} +} + +func (_c *IDeployment_LeaseID_Call) Run(run func()) *IDeployment_LeaseID_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *IDeployment_LeaseID_Call) Return(_a0 marketv1beta3.LeaseID) *IDeployment_LeaseID_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *IDeployment_LeaseID_Call) RunAndReturn(run func() marketv1beta3.LeaseID) *IDeployment_LeaseID_Call { + _c.Call.Return(run) + return _c +} + +// ManifestGroup provides a mock function with given fields: +func (_m *IDeployment) ManifestGroup() *v2beta2.Group { + ret := _m.Called() + + var r0 *v2beta2.Group + if rf, ok := ret.Get(0).(func() *v2beta2.Group); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v2beta2.Group) + } + } + + return r0 +} + +// IDeployment_ManifestGroup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ManifestGroup' +type IDeployment_ManifestGroup_Call struct { + *mock.Call +} + +// ManifestGroup is a helper method to define mock.On call +func (_e *IDeployment_Expecter) ManifestGroup() *IDeployment_ManifestGroup_Call { + return &IDeployment_ManifestGroup_Call{Call: _e.mock.On("ManifestGroup")} +} + +func (_c *IDeployment_ManifestGroup_Call) Run(run func()) *IDeployment_ManifestGroup_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *IDeployment_ManifestGroup_Call) Return(_a0 *v2beta2.Group) *IDeployment_ManifestGroup_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *IDeployment_ManifestGroup_Call) RunAndReturn(run func() *v2beta2.Group) *IDeployment_ManifestGroup_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewIDeployment interface { + mock.TestingT + Cleanup(func()) +} + +// NewIDeployment creates a new instance of IDeployment. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewIDeployment(t mockConstructorTestingTNewIDeployment) *IDeployment { + mock := &IDeployment{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/cluster/mocks/reservation.go b/cluster/mocks/reservation.go index 3cad23ee7..fc2a55640 100644 --- a/cluster/mocks/reservation.go +++ b/cluster/mocks/reservation.go @@ -63,6 +63,49 @@ func (_c *Reservation_Allocated_Call) RunAndReturn(run func() bool) *Reservation return _c } +// ClusterParams provides a mock function with given fields: +func (_m *Reservation) ClusterParams() interface{} { + ret := _m.Called() + + var r0 interface{} + if rf, ok := ret.Get(0).(func() interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// Reservation_ClusterParams_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClusterParams' +type Reservation_ClusterParams_Call struct { + *mock.Call +} + +// ClusterParams is a helper method to define mock.On call +func (_e *Reservation_Expecter) ClusterParams() *Reservation_ClusterParams_Call { + return &Reservation_ClusterParams_Call{Call: _e.mock.On("ClusterParams")} +} + +func (_c *Reservation_ClusterParams_Call) Run(run func()) *Reservation_ClusterParams_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Reservation_ClusterParams_Call) Return(_a0 interface{}) *Reservation_ClusterParams_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Reservation_ClusterParams_Call) RunAndReturn(run func() interface{}) *Reservation_ClusterParams_Call { + _c.Call.Return(run) + return _c +} + // OrderID provides a mock function with given fields: func (_m *Reservation) OrderID() marketv1beta3.OrderID { ret := _m.Called() @@ -147,6 +190,72 @@ func (_c *Reservation_Resources_Call) RunAndReturn(run func() typesv1beta3.Resou return _c } +// SetAllocatedResources provides a mock function with given fields: _a0 +func (_m *Reservation) SetAllocatedResources(_a0 []typesv1beta3.Resources) { + _m.Called(_a0) +} + +// Reservation_SetAllocatedResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetAllocatedResources' +type Reservation_SetAllocatedResources_Call struct { + *mock.Call +} + +// SetAllocatedResources is a helper method to define mock.On call +// - _a0 []typesv1beta3.Resources +func (_e *Reservation_Expecter) SetAllocatedResources(_a0 interface{}) *Reservation_SetAllocatedResources_Call { + return &Reservation_SetAllocatedResources_Call{Call: _e.mock.On("SetAllocatedResources", _a0)} +} + +func (_c *Reservation_SetAllocatedResources_Call) Run(run func(_a0 []typesv1beta3.Resources)) *Reservation_SetAllocatedResources_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]typesv1beta3.Resources)) + }) + return _c +} + +func (_c *Reservation_SetAllocatedResources_Call) Return() *Reservation_SetAllocatedResources_Call { + _c.Call.Return() + return _c +} + +func (_c *Reservation_SetAllocatedResources_Call) RunAndReturn(run func([]typesv1beta3.Resources)) *Reservation_SetAllocatedResources_Call { + _c.Call.Return(run) + return _c +} + +// SetClusterParams provides a mock function with given fields: _a0 +func (_m *Reservation) SetClusterParams(_a0 interface{}) { + _m.Called(_a0) +} + +// Reservation_SetClusterParams_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetClusterParams' +type Reservation_SetClusterParams_Call struct { + *mock.Call +} + +// SetClusterParams is a helper method to define mock.On call +// - _a0 interface{} +func (_e *Reservation_Expecter) SetClusterParams(_a0 interface{}) *Reservation_SetClusterParams_Call { + return &Reservation_SetClusterParams_Call{Call: _e.mock.On("SetClusterParams", _a0)} +} + +func (_c *Reservation_SetClusterParams_Call) Run(run func(_a0 interface{})) *Reservation_SetClusterParams_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(interface{})) + }) + return _c +} + +func (_c *Reservation_SetClusterParams_Call) Return() *Reservation_SetClusterParams_Call { + _c.Call.Return() + return _c +} + +func (_c *Reservation_SetClusterParams_Call) RunAndReturn(run func(interface{})) *Reservation_SetClusterParams_Call { + _c.Call.Return(run) + return _c +} + type mockConstructorTestingTNewReservation interface { mock.TestingT Cleanup(func()) diff --git a/cluster/mocks/reservation_group.go b/cluster/mocks/reservation_group.go new file mode 100644 index 000000000..c5691f4a4 --- /dev/null +++ b/cluster/mocks/reservation_group.go @@ -0,0 +1,188 @@ +// Code generated by mockery v2.24.0. DO NOT EDIT. + +package mocks + +import ( + typesv1beta3 "github.com/akash-network/akash-api/go/node/types/v1beta3" + mock "github.com/stretchr/testify/mock" +) + +// ReservationGroup is an autogenerated mock type for the ReservationGroup type +type ReservationGroup struct { + mock.Mock +} + +type ReservationGroup_Expecter struct { + mock *mock.Mock +} + +func (_m *ReservationGroup) EXPECT() *ReservationGroup_Expecter { + return &ReservationGroup_Expecter{mock: &_m.Mock} +} + +// ClusterParams provides a mock function with given fields: +func (_m *ReservationGroup) ClusterParams() interface{} { + ret := _m.Called() + + var r0 interface{} + if rf, ok := ret.Get(0).(func() interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// ReservationGroup_ClusterParams_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClusterParams' +type ReservationGroup_ClusterParams_Call struct { + *mock.Call +} + +// ClusterParams is a helper method to define mock.On call +func (_e *ReservationGroup_Expecter) ClusterParams() *ReservationGroup_ClusterParams_Call { + return &ReservationGroup_ClusterParams_Call{Call: _e.mock.On("ClusterParams")} +} + +func (_c *ReservationGroup_ClusterParams_Call) Run(run func()) *ReservationGroup_ClusterParams_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ReservationGroup_ClusterParams_Call) Return(_a0 interface{}) *ReservationGroup_ClusterParams_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ReservationGroup_ClusterParams_Call) RunAndReturn(run func() interface{}) *ReservationGroup_ClusterParams_Call { + _c.Call.Return(run) + return _c +} + +// Resources provides a mock function with given fields: +func (_m *ReservationGroup) Resources() typesv1beta3.ResourceGroup { + ret := _m.Called() + + var r0 typesv1beta3.ResourceGroup + if rf, ok := ret.Get(0).(func() typesv1beta3.ResourceGroup); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(typesv1beta3.ResourceGroup) + } + } + + return r0 +} + +// ReservationGroup_Resources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Resources' +type ReservationGroup_Resources_Call struct { + *mock.Call +} + +// Resources is a helper method to define mock.On call +func (_e *ReservationGroup_Expecter) Resources() *ReservationGroup_Resources_Call { + return &ReservationGroup_Resources_Call{Call: _e.mock.On("Resources")} +} + +func (_c *ReservationGroup_Resources_Call) Run(run func()) *ReservationGroup_Resources_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ReservationGroup_Resources_Call) Return(_a0 typesv1beta3.ResourceGroup) *ReservationGroup_Resources_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ReservationGroup_Resources_Call) RunAndReturn(run func() typesv1beta3.ResourceGroup) *ReservationGroup_Resources_Call { + _c.Call.Return(run) + return _c +} + +// SetAllocatedResources provides a mock function with given fields: _a0 +func (_m *ReservationGroup) SetAllocatedResources(_a0 []typesv1beta3.Resources) { + _m.Called(_a0) +} + +// ReservationGroup_SetAllocatedResources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetAllocatedResources' +type ReservationGroup_SetAllocatedResources_Call struct { + *mock.Call +} + +// SetAllocatedResources is a helper method to define mock.On call +// - _a0 []typesv1beta3.Resources +func (_e *ReservationGroup_Expecter) SetAllocatedResources(_a0 interface{}) *ReservationGroup_SetAllocatedResources_Call { + return &ReservationGroup_SetAllocatedResources_Call{Call: _e.mock.On("SetAllocatedResources", _a0)} +} + +func (_c *ReservationGroup_SetAllocatedResources_Call) Run(run func(_a0 []typesv1beta3.Resources)) *ReservationGroup_SetAllocatedResources_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]typesv1beta3.Resources)) + }) + return _c +} + +func (_c *ReservationGroup_SetAllocatedResources_Call) Return() *ReservationGroup_SetAllocatedResources_Call { + _c.Call.Return() + return _c +} + +func (_c *ReservationGroup_SetAllocatedResources_Call) RunAndReturn(run func([]typesv1beta3.Resources)) *ReservationGroup_SetAllocatedResources_Call { + _c.Call.Return(run) + return _c +} + +// SetClusterParams provides a mock function with given fields: _a0 +func (_m *ReservationGroup) SetClusterParams(_a0 interface{}) { + _m.Called(_a0) +} + +// ReservationGroup_SetClusterParams_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetClusterParams' +type ReservationGroup_SetClusterParams_Call struct { + *mock.Call +} + +// SetClusterParams is a helper method to define mock.On call +// - _a0 interface{} +func (_e *ReservationGroup_Expecter) SetClusterParams(_a0 interface{}) *ReservationGroup_SetClusterParams_Call { + return &ReservationGroup_SetClusterParams_Call{Call: _e.mock.On("SetClusterParams", _a0)} +} + +func (_c *ReservationGroup_SetClusterParams_Call) Run(run func(_a0 interface{})) *ReservationGroup_SetClusterParams_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(interface{})) + }) + return _c +} + +func (_c *ReservationGroup_SetClusterParams_Call) Return() *ReservationGroup_SetClusterParams_Call { + _c.Call.Return() + return _c +} + +func (_c *ReservationGroup_SetClusterParams_Call) RunAndReturn(run func(interface{})) *ReservationGroup_SetClusterParams_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewReservationGroup interface { + mock.TestingT + Cleanup(func()) +} + +// NewReservationGroup creates a new instance of ReservationGroup. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewReservationGroup(t mockConstructorTestingTNewReservationGroup) *ReservationGroup { + mock := &ReservationGroup{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/cluster/monitor.go b/cluster/monitor.go index bd626aa98..5a4e8e2ad 100644 --- a/cluster/monitor.go +++ b/cluster/monitor.go @@ -11,11 +11,11 @@ import ( "github.com/boz/go-lifecycle" "github.com/tendermint/tendermint/libs/log" - manifest "github.com/akash-network/akash-api/go/manifest/v2beta2" mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" "github.com/akash-network/node/pubsub" "github.com/akash-network/node/util/runner" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" "github.com/akash-network/provider/cluster/util" "github.com/akash-network/provider/event" "github.com/akash-network/provider/session" @@ -41,8 +41,7 @@ type deploymentMonitor struct { session session.Session client Client - lease mtypes.LeaseID - mgroup *manifest.Group + deployment ctypes.IDeployment attempts int log log.Logger @@ -56,8 +55,7 @@ func newDeploymentMonitor(dm *deploymentManager) *deploymentMonitor { bus: dm.bus, session: dm.session, client: dm.client, - lease: dm.lease, - mgroup: dm.mgroup, + deployment: dm.deployment, log: dm.log.With("cmp", "deployment-monitor"), lc: lifecycle.New(), clusterSettings: dm.config.ClusterSettings, @@ -168,7 +166,7 @@ func (m *deploymentMonitor) runCheck(ctx context.Context) <-chan runner.Result { func (m *deploymentMonitor) doCheck(ctx context.Context) (bool, error) { clientCtx := util.ApplyToContext(ctx, m.clusterSettings) - status, err := m.client.LeaseStatus(clientCtx, m.lease) + status, err := m.client.LeaseStatus(clientCtx, m.deployment.LeaseID()) if err != nil { m.log.Error("lease status", "err", err) @@ -177,7 +175,7 @@ func (m *deploymentMonitor) doCheck(ctx context.Context) (bool, error) { badsvc := 0 - for _, spec := range m.mgroup.Services { + for _, spec := range m.deployment.ManifestGroup().Services { service, foundService := status[spec.Name] if foundService { if uint32(service.Available) < spec.Count { @@ -203,7 +201,7 @@ func (m *deploymentMonitor) runCloseLease(ctx context.Context) <-chan runner.Res return runner.Do(func() runner.Result { // TODO: retry, timeout err := m.session.Client().Tx().Broadcast(ctx, &mtypes.MsgCloseBid{ - BidID: m.lease.BidID(), + BidID: m.deployment.LeaseID().BidID(), }) if err != nil { m.log.Error("closing deployment", "err", err) @@ -216,8 +214,8 @@ func (m *deploymentMonitor) runCloseLease(ctx context.Context) <-chan runner.Res func (m *deploymentMonitor) publishStatus(status event.ClusterDeploymentStatus) { if err := m.bus.Publish(event.ClusterDeployment{ - LeaseID: m.lease, - Group: m.mgroup, + LeaseID: m.deployment.LeaseID(), + Group: m.deployment.ManifestGroup(), Status: status, }); err != nil { m.log.Error("publishing manifest group deployed event", "err", err, "status", status) diff --git a/cluster/monitor_test.go b/cluster/monitor_test.go index 181652af3..08f0d487e 100644 --- a/cluster/monitor_test.go +++ b/cluster/monitor_test.go @@ -20,24 +20,25 @@ import ( func TestMonitorInstantiate(t *testing.T) { myLog := testutil.Logger(t) bus := pubsub.NewBus() - lid := testutil.LeaseID(t) - group := &manifest.Group{} client := &mocks.Client{} + deployment := &ctypes.Deployment{ + Lid: testutil.LeaseID(t), + MGroup: &manifest.Group{}, + } statusResult := &ctypes.LeaseStatus{} - client.On("LeaseStatus", mock.Anything, lid).Return(statusResult, nil) + client.On("LeaseStatus", mock.Anything, deployment.LeaseID()).Return(statusResult, nil) mySession := session.New(myLog, nil, nil, -1) lc := lifecycle.New() myDeploymentManager := &deploymentManager{ - bus: bus, - session: mySession, - client: client, - lease: lid, - mgroup: group, - log: myLog, - lc: lc, + bus: bus, + session: mySession, + client: client, + deployment: deployment, + log: myLog, + lc: lc, } monitor := newDeploymentMonitor(myDeploymentManager) require.NotNil(t, monitor) @@ -49,7 +50,6 @@ func TestMonitorSendsClusterDeploymentPending(t *testing.T) { const serviceName = "test" myLog := testutil.Logger(t) bus := pubsub.NewBus() - lid := testutil.LeaseID(t) group := &manifest.Group{} group.Services = make(manifest.Services, 1) @@ -59,29 +59,32 @@ func TestMonitorSendsClusterDeploymentPending(t *testing.T) { group.Services[0].Expose[0].Proto = manifest.TCP group.Services[0].Expose[0].Port = 40000 client := &mocks.Client{} + deployment := &ctypes.Deployment{ + Lid: testutil.LeaseID(t), + MGroup: group, + } statusResult := make(map[string]*ctypes.ServiceStatus) - client.On("LeaseStatus", mock.Anything, lid).Return(statusResult, nil) + client.On("LeaseStatus", mock.Anything, deployment.LeaseID()).Return(statusResult, nil) mySession := session.New(myLog, nil, nil, -1) sub, err := bus.Subscribe() require.NoError(t, err) lc := lifecycle.New() myDeploymentManager := &deploymentManager{ - bus: bus, - session: mySession, - client: client, - lease: lid, - mgroup: group, - log: myLog, - lc: lc, + bus: bus, + session: mySession, + client: client, + deployment: deployment, + log: myLog, + lc: lc, } monitor := newDeploymentMonitor(myDeploymentManager) require.NotNil(t, monitor) ev := <-sub.Events() result := ev.(event.ClusterDeployment) - require.Equal(t, lid, result.LeaseID) + require.Equal(t, deployment.LeaseID(), result.LeaseID) require.Equal(t, event.ClusterDeploymentPending, result.Status) monitor.lc.Shutdown(nil) @@ -91,7 +94,6 @@ func TestMonitorSendsClusterDeploymentDeployed(t *testing.T) { const serviceName = "test" myLog := testutil.Logger(t) bus := pubsub.NewBus() - lid := testutil.LeaseID(t) group := &manifest.Group{} group.Services = make(manifest.Services, 1) @@ -102,6 +104,10 @@ func TestMonitorSendsClusterDeploymentDeployed(t *testing.T) { group.Services[0].Expose[0].Port = 40000 group.Services[0].Count = 3 client := &mocks.Client{} + deployment := &ctypes.Deployment{ + Lid: testutil.LeaseID(t), + MGroup: group, + } statusResult := make(map[string]*ctypes.ServiceStatus) statusResult[serviceName] = &ctypes.ServiceStatus{ @@ -115,27 +121,26 @@ func TestMonitorSendsClusterDeploymentDeployed(t *testing.T) { ReadyReplicas: 0, AvailableReplicas: 0, } - client.On("LeaseStatus", mock.Anything, lid).Return(statusResult, nil) + client.On("LeaseStatus", mock.Anything, deployment.LeaseID()).Return(statusResult, nil) mySession := session.New(myLog, nil, nil, -1) sub, err := bus.Subscribe() require.NoError(t, err) lc := lifecycle.New() myDeploymentManager := &deploymentManager{ - bus: bus, - session: mySession, - client: client, - lease: lid, - mgroup: group, - log: myLog, - lc: lc, + bus: bus, + session: mySession, + client: client, + deployment: deployment, + log: myLog, + lc: lc, } monitor := newDeploymentMonitor(myDeploymentManager) require.NotNil(t, monitor) ev := <-sub.Events() result := ev.(event.ClusterDeployment) - require.Equal(t, lid, result.LeaseID) + require.Equal(t, deployment.LeaseID(), result.LeaseID) require.Equal(t, event.ClusterDeploymentDeployed, result.Status) monitor.lc.Shutdown(nil) diff --git a/cluster/reservation.go b/cluster/reservation.go index fdc155989..14ff909aa 100644 --- a/cluster/reservation.go +++ b/cluster/reservation.go @@ -16,11 +16,13 @@ func newReservation(order mtypes.OrderID, resources atypes.ResourceGroup) *reser } type reservation struct { - order mtypes.OrderID - resources atypes.ResourceGroup - allocated bool - endpointQuantity uint - ipsConfirmed bool + order mtypes.OrderID + resources atypes.ResourceGroup + adjustedResources []atypes.Resources + clusterParams interface{} + endpointQuantity uint + allocated bool + ipsConfirmed bool } var _ ctypes.Reservation = (*reservation)(nil) @@ -33,6 +35,18 @@ func (r *reservation) Resources() atypes.ResourceGroup { return r.resources } +func (r *reservation) SetAllocatedResources(val []atypes.Resources) { + r.adjustedResources = val +} + +func (r *reservation) SetClusterParams(val interface{}) { + r.clusterParams = val +} + +func (r *reservation) ClusterParams() interface{} { + return r.clusterParams +} + func (r *reservation) Allocated() bool { return r.allocated } diff --git a/cluster/service.go b/cluster/service.go index 25a9b9958..0930cfd27 100644 --- a/cluster/service.go +++ b/cluster/service.go @@ -262,7 +262,7 @@ func (s *service) updateDeploymentManagerGauge() { deploymentManagerGauge.Set(float64(len(s.managers))) } -func (s *service) run(ctx context.Context, deployments []ctypes.Deployment) { +func (s *service) run(ctx context.Context, deployments []ctypes.IDeployment) { defer s.lc.ShutdownCompleted() defer s.sub.Close() @@ -276,9 +276,7 @@ func (s *service) run(ctx context.Context, deployments []ctypes.Deployment) { } for _, deployment := range deployments { - key := deployment.LeaseID() - mgroup := deployment.ManifestGroup() - s.managers[key] = newDeploymentManager(s, deployment.LeaseID(), &mgroup, false) + s.managers[deployment.LeaseID()] = newDeploymentManager(s, deployment, false) s.updateDeploymentManagerGauge() } @@ -299,20 +297,28 @@ loop: break } - if _, err := s.inventory.lookup(ev.LeaseID.OrderID(), mgroup); err != nil { + reservation, err := s.inventory.lookup(ev.LeaseID.OrderID(), mgroup) + + if err != nil { s.log.Error("error looking up manifest", "err", err, "lease", ev.LeaseID, "group-name", mgroup.Name) break } + deployment := &ctypes.Deployment{ + Lid: ev.LeaseID, + MGroup: mgroup, + CParams: reservation.ClusterParams(), + } + key := ev.LeaseID if manager := s.managers[key]; manager != nil { - if err := manager.update(mgroup); err != nil { + if err := manager.update(deployment); err != nil { s.log.Error("updating deployment", "err", err, "lease", ev.LeaseID, "group-name", mgroup.Name) } break } - s.managers[key] = newDeploymentManager(s, ev.LeaseID, mgroup, true) + s.managers[key] = newDeploymentManager(s, deployment, true) case mtypes.EventLeaseClosed: _ = s.bus.Publish(event.LeaseRemoveFundsMonitor{LeaseID: ev.ID}) s.teardownLease(ev.ID) @@ -322,15 +328,16 @@ loop: Leases: uint32(len(s.managers)), } case dm := <-s.managerch: - s.log.Info("manager done", "lease", dm.lease) + s.log.Info("manager done", "lease", dm.deployment.LeaseID()) // unreserve resources - if err := s.inventory.unreserve(dm.lease.OrderID()); err != nil { - s.log.Error("unreserving inventory", "err", err, - "lease", dm.lease) + if err := s.inventory.unreserve(dm.deployment.LeaseID().OrderID()); err != nil { + s.log.Error("unreserving inventory", + "err", err, + "lease", dm.deployment.LeaseID()) } - delete(s.managers, dm.lease) + delete(s.managers, dm.deployment.LeaseID()) case req := <-s.checkDeploymentExistsRequestCh: s.doCheckDeploymentExists(req) } @@ -340,7 +347,7 @@ loop: for _, manager := range s.managers { if manager != nil { manager := <-s.managerch - s.log.Debug("manager done", "lease", manager.lease) + s.log.Debug("manager done", "lease", manager.deployment.LeaseID()) } } @@ -378,7 +385,7 @@ func (s *service) teardownLease(lid mtypes.LeaseID) { } } -func findDeployments(ctx context.Context, log log.Logger, client Client, _ session.Session) ([]ctypes.Deployment, error) { +func findDeployments(ctx context.Context, log log.Logger, client Client, _ session.Session) ([]ctypes.IDeployment, error) { deployments, err := client.Deployments(ctx) if err != nil { log.Error("fetching deployments", "err", err) diff --git a/cluster/types/v1beta3/deployment.go b/cluster/types/v1beta3/deployment.go index 59a836780..cb65632e3 100644 --- a/cluster/types/v1beta3/deployment.go +++ b/cluster/types/v1beta3/deployment.go @@ -5,86 +5,31 @@ import ( mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" ) -// Deployment interface defined with LeaseID and ManifestGroup methods +// IDeployment interface defined with LeaseID and ManifestGroup methods // -//go:generate mockery --name Deployment --output ../../mocks -type Deployment interface { +//go:generate mockery --name IDeployment --output ../../mocks +type IDeployment interface { LeaseID() mtypes.LeaseID - ManifestGroup() maniv2beta2.Group + ManifestGroup() *maniv2beta2.Group + ClusterParams() interface{} } -// // ParamsLease provider specific parameters required by deployment for correct operation -// type LeaseServiceParams struct { -// RuntimeClass string `json:"runtime_class"` -// } +type Deployment struct { + Lid mtypes.LeaseID + MGroup *maniv2beta2.Group + CParams interface{} +} -// // Service extends manifest service with provider specific parameters -// type Service struct { -// maniv2beta2.Service `json:",inline"` -// ProviderParams ServiceProviderParams `json:"provider_params"` -// } -// -// type Services []Service -// -// type Group struct { -// Name string -// Services Services -// } -// -// type PGroup interface { -// maniv2beta2.IGroup -// ProviderGroup() *Group -// } -// -// var _ types.ResourceGroup = (*Group)(nil) -// var _ PGroup = (*Group)(nil) -// -// // GetName returns the name of group -// func (g Group) GetName() string { -// return g.Name -// } -// -// // GetResources returns list of resources in a group -// func (g Group) GetResources() []types.Resources { -// resources := make([]types.Resources, 0, len(g.Services)) -// for _, s := range g.Services { -// resources = append(resources, types.Resources{ -// Resources: s.Resources, -// Count: s.Count, -// }) -// } -// -// return resources -// } -// -// func (g Group) ProviderGroup() *Group { -// return &g -// } -// -// func (g Group) ManifestGroup() *maniv2beta2.Group { -// mgroup := &maniv2beta2.Group{ -// Name: g.Name, -// Services: make(maniv2beta2.Services, 0, len(g.Services)), -// } -// -// for _, svc := range g.Services { -// mgroup.Services = append(mgroup.Services, svc.Service) -// } -// -// return mgroup -// } -// -// func ProviderGroupFromManifest(mani *maniv2beta2.Group) *Group { -// g := &Group{ -// Name: mani.Name, -// Services: make(Services, 0, len(mani.Services)), -// } -// -// for _, svc := range mani.Services { -// g.Services = append(g.Services, Service{ -// Service: svc, -// }) -// } -// -// return g -// } +var _ IDeployment = (*Deployment)(nil) + +func (d *Deployment) LeaseID() mtypes.LeaseID { + return d.Lid +} + +func (d *Deployment) ManifestGroup() *maniv2beta2.Group { + return d.MGroup +} + +func (d *Deployment) ClusterParams() interface{} { + return d.CParams +} diff --git a/cluster/types/v1beta3/reservation.go b/cluster/types/v1beta3/reservation.go index bc8eb2ad6..ea6a6cb81 100644 --- a/cluster/types/v1beta3/reservation.go +++ b/cluster/types/v1beta3/reservation.go @@ -5,11 +5,19 @@ import ( atypes "github.com/akash-network/akash-api/go/node/types/v1beta3" ) +//go:generate mockery --name ReservationGroup --output ../../mocks +type ReservationGroup interface { + Resources() atypes.ResourceGroup + SetAllocatedResources([]atypes.Resources) + SetClusterParams(interface{}) + ClusterParams() interface{} +} + // Reservation interface implements orders and resources // //go:generate mockery --name Reservation --output ../../mocks type Reservation interface { OrderID() mtypes.OrderID - Resources() atypes.ResourceGroup Allocated() bool + ReservationGroup } diff --git a/cluster/types/v1beta3/types.go b/cluster/types/v1beta3/types.go index 2fbabf04e..75e7a37d8 100644 --- a/cluster/types/v1beta3/types.go +++ b/cluster/types/v1beta3/types.go @@ -3,13 +3,14 @@ package v1beta3 import ( "bufio" "context" + "fmt" "io" + "strings" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/pkg/errors" eventsv1 "k8s.io/api/events/v1" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/akash-network/node/sdl" manifest "github.com/akash-network/akash-api/go/manifest/v2beta2" @@ -58,6 +59,70 @@ type InventoryNodeMetric struct { StorageEphemeral uint64 `json:"storage_ephemeral"` } +type GPUAttributes map[string][]string + +type StorageAttributes struct { + Persistent bool `json:"persistent"` + Class string `json:"class,omitempty"` +} + +func ParseGPUAttributes(attrs types.Attributes) (GPUAttributes, error) { + var nvidia []string + var amd []string + + for _, attr := range attrs { + tokens := strings.Split(attr.Key, "/") + if len(tokens) != 4 { + return GPUAttributes{}, fmt.Errorf("invalid GPU attribute") // nolint: goerr113 + } + + switch tokens[0] { + case "vendor": + default: + return GPUAttributes{}, fmt.Errorf("unexpected GPU attribute type (%s)", tokens[0]) // nolint: goerr113 + } + + switch tokens[1] { + case "nvidia": + nvidia = append(nvidia, tokens[3]) + case "amd": + amd = append(amd, tokens[3]) + default: + return GPUAttributes{}, fmt.Errorf("unsupported GPU vendor (%s)", tokens[1]) // nolint: goerr113 + } + + } + + res := make(GPUAttributes) + if len(nvidia) > 0 { + res["nvidia"] = nvidia + } + + if len(amd) > 0 { + res["amd"] = amd + } + + return res, nil +} + +func ParseStorageAttributes(attrs types.Attributes) (StorageAttributes, error) { + attr := attrs.Find(sdl.StorageAttributePersistent) + persistent, _ := attr.AsBool() + attr = attrs.Find(sdl.StorageAttributeClass) + class, _ := attr.AsString() + + if persistent && class == "" { + return StorageAttributes{}, fmt.Errorf("persistent volume must specify storage class") // nolint: goerr113 + } + + res := StorageAttributes{ + Persistent: persistent, + Class: class, + } + + return res, nil +} + func (inv *InventoryMetricTotal) AddResources(res types.Resources) { cpu := sdk.NewIntFromUint64(inv.CPU) gpu := sdk.NewIntFromUint64(inv.GPU) @@ -132,8 +197,21 @@ type LeaseStatus struct { ForwardedPorts map[string][]ForwardedPortStatus `json:"forwarded_ports"` // Container services that are externally accessible } +type InventoryOptions struct { + DryRun bool +} + +type InventoryOption func(*InventoryOptions) *InventoryOptions + +func WithDryRun() InventoryOption { + return func(opts *InventoryOptions) *InventoryOptions { + opts.DryRun = true + return opts + } +} + type Inventory interface { - Adjust(Reservation) error + Adjust(ReservationGroup, ...InventoryOption) error Metrics() InventoryMetrics } diff --git a/gateway/rest/integration_test.go b/gateway/rest/integration_test.go index 52d842a52..2c517ebcb 100644 --- a/gateway/rest/integration_test.go +++ b/gateway/rest/integration_test.go @@ -67,9 +67,10 @@ func Test_router_Validate(t *testing.T) { } addr := testutil.AccAddress(t) mocks := createMocks() - mocks.pclient.On("Validate", mock.Anything, mock.Anything).Return(expected, nil) + mocks.pclient.On("Validate", mock.Anything, mock.Anything, mock.Anything).Return(expected, nil) withServer(t, addr, mocks.pclient, mocks.qclient, nil, operatorclients.NullIPOperatorClient(), func(host string) { - client, err := NewClient(mocks.qclient, addr, nil) + cert := testutil.Certificate(t, testutil.AccAddress(t), testutil.CertificateOptionMocks(mocks.qclient)) + client, err := NewClient(mocks.qclient, addr, cert.Cert) assert.NoError(t, err) result, err := client.Validate(context.Background(), testutil.GroupSpec(t)) assert.NoError(t, err) @@ -81,9 +82,10 @@ func Test_router_Validate(t *testing.T) { t.Run("failure", func(t *testing.T) { addr := testutil.AccAddress(t) mocks := createMocks() - mocks.pclient.On("Validate", mock.Anything, mock.Anything).Return(provider.ValidateGroupSpecResult{}, errors.New("oops")) + mocks.pclient.On("Validate", mock.Anything, mock.Anything, mock.Anything).Return(provider.ValidateGroupSpecResult{}, errors.New("oops")) withServer(t, addr, mocks.pclient, mocks.qclient, nil, operatorclients.NullIPOperatorClient(), func(host string) { - client, err := NewClient(mocks.qclient, addr, nil) + cert := testutil.Certificate(t, testutil.AccAddress(t), testutil.CertificateOptionMocks(mocks.qclient)) + client, err := NewClient(mocks.qclient, addr, cert.Cert) assert.NoError(t, err) _, err = client.Validate(context.Background(), dtypes.GroupSpec{}) assert.Error(t, err) diff --git a/gateway/rest/router.go b/gateway/rest/router.go index dda346099..d0f5afbe6 100644 --- a/gateway/rest/router.go +++ b/gateway/rest/router.go @@ -107,9 +107,18 @@ func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ipopcl createStatusHandler(log, pclient, addr)). Methods("GET") + vrouter := router.NewRoute().Subrouter() + vrouter.Use(requireOwner()) + // GET /validate // validate endpoint checks if provider will bid on given groupspec - router.HandleFunc("/validate", + vrouter.HandleFunc("/validate", + validateHandler(log, pclient)). + Methods("GET") + + // GET /wiboy (aka would I bid on you) + // validate endpoint checks if provider will bid on given groupspec + vrouter.HandleFunc("/wiboy", validateHandler(log, pclient)). Methods("GET") @@ -528,6 +537,8 @@ func validateHandler(log log.Logger, cl provider.ValidateClient) http.HandlerFun return } + owner := requestOwner(req) + var gspec dtypes.GroupSpec if err := json.Unmarshal(data, &gspec); err != nil { @@ -535,7 +546,7 @@ func validateHandler(log log.Logger, cl provider.ValidateClient) http.HandlerFun return } - validate, err := cl.Validate(req.Context(), gspec) + validate, err := cl.Validate(req.Context(), owner, gspec) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/gateway/rest/router_migrate_test.go b/gateway/rest/router_migrate_test.go index ef6241384..cd0a82801 100644 --- a/gateway/rest/router_migrate_test.go +++ b/gateway/rest/router_migrate_test.go @@ -24,7 +24,7 @@ func TestRouteMigrateHostnameDoesNotExist(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - err := test.gclient.MigrateHostnames(ctx, []string{"foobar.com"}, dseq, gseq) + err := test.gwclient.MigrateHostnames(ctx, []string{"foobar.com"}, dseq, gseq) require.Error(t, err) require.IsType(t, ClientResponseError{}, err) require.Regexp(t, `(?s)^.*destination deployment does not exist.*$`, err.(ClientResponseError).ClientError()) @@ -43,7 +43,7 @@ func TestRouteMigrateHostnameDeploymentDoesNotUse(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - err := test.gclient.MigrateHostnames(ctx, []string{"foobar.org"}, dseq, gseq) + err := test.gwclient.MigrateHostnames(ctx, []string{"foobar.org"}, dseq, gseq) require.Error(t, err) require.IsType(t, ClientResponseError{}, err) require.Regexp(t, `(?s)^.*the hostname "foobar.org" is not used by this deployment.*$`, err.(ClientResponseError).ClientError()) @@ -87,7 +87,7 @@ func TestRouteMigrateHostname(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - err := test.gclient.MigrateHostnames(ctx, []string{hostname}, dseq, gseq) + err := test.gwclient.MigrateHostnames(ctx, []string{hostname}, dseq, gseq) require.NoError(t, err) require.Equal(t, 2, len(test.clusterService.Calls)) @@ -131,7 +131,7 @@ func TestRouteMigrateHostnamePrepareFails(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - err := test.gclient.MigrateHostnames(ctx, []string{hostname}, dseq, gseq) + err := test.gwclient.MigrateHostnames(ctx, []string{hostname}, dseq, gseq) require.Error(t, err) require.Regexp(t, `^.*remote server returned 500.*$`, err) require.Equal(t, 1, len(test.clusterService.Calls)) @@ -177,7 +177,7 @@ func TestRouteMigrateHostnameTransferFails(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - err := test.gclient.MigrateHostnames(ctx, []string{hostname}, dseq, gseq) + err := test.gwclient.MigrateHostnames(ctx, []string{hostname}, dseq, gseq) require.Error(t, err) require.Regexp(t, `^.*remote server returned 500.*$`, err) require.Equal(t, 2, len(test.clusterService.Calls)) diff --git a/gateway/rest/router_test.go b/gateway/rest/router_test.go index d6eee817d..824359ef5 100644 --- a/gateway/rest/router_test.go +++ b/gateway/rest/router_test.go @@ -67,7 +67,7 @@ type routerTest struct { qclient *qmock.QueryClient clusterService *pcmock.Service hostnameClient *pcmock.HostnameServiceClient - gclient *client + gwclient *client ccert testutil.TestCertificate pcert testutil.TestCertificate host *url.URL @@ -112,7 +112,7 @@ func runRouterTest(t *testing.T, authClient bool, fn func(*routerTest)) { require.NoError(t, err) require.NotNil(t, gclient) - mf.gclient = gclient.(*client) + mf.gwclient = gclient.(*client) fn(mf) }) @@ -145,7 +145,7 @@ func testCertHelper(t *testing.T, test *routerTest) { req.Header.Set("Content-Type", contentTypeJSON) - _, err = test.gclient.hclient.Do(req) + _, err = test.gwclient.hclient.Do(req) require.Error(t, err) // return error message looks like // Put "https://127.0.0.1:58536/deployment/652/manifest": tls: unable to verify certificate: x509: cannot validate certificate for 127.0.0.1 because it doesn't contain any IP SANs @@ -181,7 +181,7 @@ func TestRouteNotActiveClientCert(t *testing.T) { require.NoError(t, err) require.NotNil(t, gclient) - mf.gclient = gclient.(*client) + mf.gwclient = gclient.(*client) testCertHelper(t, mf) }) @@ -217,7 +217,7 @@ func TestRouteExpiredClientCert(t *testing.T) { require.NoError(t, err) require.NotNil(t, gclient) - mf.gclient = gclient.(*client) + mf.gwclient = gclient.(*client) testCertHelper(t, mf) }) @@ -256,7 +256,7 @@ func TestRouteNotActiveServerCert(t *testing.T) { require.NoError(t, err) require.NotNil(t, gclient) - mf.gclient = gclient.(*client) + mf.gwclient = gclient.(*client) testCertHelper(t, mf) }) @@ -296,7 +296,7 @@ func TestRouteExpiredServerCert(t *testing.T) { require.NoError(t, err) require.NotNil(t, gclient) - mf.gclient = gclient.(*client) + mf.gwclient = gclient.(*client) testCertHelper(t, mf) }) @@ -312,9 +312,9 @@ func TestRouteDoesNotExist(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusNotFound) + require.Equal(t, http.StatusNotFound, resp.StatusCode) }) } @@ -356,9 +356,9 @@ func TestRouteVersionOK(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, resp.StatusCode) var data versionInfo decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&data) @@ -389,9 +389,9 @@ func TestRouteStatusOK(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, resp.StatusCode) data := make(map[string]interface{}) decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&data) @@ -413,9 +413,9 @@ func TestRouteStatusFails(t *testing.T) { require.NoError(t, err) req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusInternalServerError) + require.Equal(t, http.StatusInternalServerError, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) @@ -424,12 +424,12 @@ func TestRouteStatusFails(t *testing.T) { } func TestRouteValidateOK(t *testing.T) { - runRouterTest(t, false, func(test *routerTest) { + runRouterTest(t, true, func(test *routerTest) { validate := provider.ValidateGroupSpecResult{ MinBidPrice: testutil.AkashDecCoin(t, 200), } - test.pclient.On("Validate", mock.Anything, mock.Anything).Return(validate, nil) + test.pclient.On("Validate", mock.Anything, mock.Anything, mock.Anything).Return(validate, nil) uri, err := makeURI(test.host, validatePath()) require.NoError(t, err) @@ -443,9 +443,9 @@ func TestRouteValidateOK(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, resp.StatusCode) data := make(map[string]interface{}) decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&data) @@ -453,9 +453,13 @@ func TestRouteValidateOK(t *testing.T) { }) } -func TestRouteValidateFails(t *testing.T) { +func TestRouteValidateUnauthorized(t *testing.T) { runRouterTest(t, false, func(test *routerTest) { - test.pclient.On("Validate", mock.Anything, mock.Anything).Return(provider.ValidateGroupSpecResult{}, errGeneric) + validate := provider.ValidateGroupSpecResult{ + MinBidPrice: testutil.AkashDecCoin(t, 200), + } + + test.pclient.On("Validate", mock.Anything, mock.Anything, mock.Anything).Return(validate, nil) uri, err := makeURI(test.host, validatePath()) require.NoError(t, err) @@ -468,9 +472,31 @@ func TestRouteValidateFails(t *testing.T) { require.NoError(t, err) req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusInternalServerError) + require.Equal(t, http.StatusUnauthorized, resp.StatusCode) + }) +} + +func TestRouteValidateFails(t *testing.T) { + runRouterTest(t, true, func(test *routerTest) { + test.pclient.On("Validate", mock.Anything, mock.Anything, mock.Anything).Return(provider.ValidateGroupSpecResult{}, errGeneric) + + uri, err := makeURI(test.host, validatePath()) + require.NoError(t, err) + + gspec := testutil.GroupSpec(t) + bgspec, err := json.Marshal(&gspec) + require.NoError(t, err) + + req, err := http.NewRequest("GET", uri, bytes.NewReader(bgspec)) + require.NoError(t, err) + + req.Header.Set("Content-Type", contentTypeJSON) + resp, err := test.gwclient.hclient.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusInternalServerError, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) @@ -479,7 +505,7 @@ func TestRouteValidateFails(t *testing.T) { } func TestRouteValidateFailsEmptyBody(t *testing.T) { - runRouterTest(t, false, func(test *routerTest) { + runRouterTest(t, true, func(test *routerTest) { test.pclient.On("Validate", mock.Anything, mock.Anything).Return(provider.ValidateGroupSpecResult{}, errGeneric) uri, err := makeURI(test.host, validatePath()) @@ -489,9 +515,9 @@ func TestRouteValidateFailsEmptyBody(t *testing.T) { require.NoError(t, err) req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusBadRequest) + require.Equal(t, http.StatusBadRequest, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) @@ -529,9 +555,9 @@ func TestRoutePutManifestOK(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) @@ -570,9 +596,9 @@ func TestRoutePutInvalidManifest(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusUnprocessableEntity) + require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) @@ -657,9 +683,9 @@ func TestRouteLeaseStatusOk(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, resp.StatusCode) data := make(map[string]interface{}) dec := json.NewDecoder(resp.Body) @@ -705,9 +731,9 @@ func TestRouteLeaseNotInKubernetes(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusNotFound) + require.Equal(t, http.StatusNotFound, resp.StatusCode) }) } @@ -727,10 +753,10 @@ func TestRouteLeaseStatusErr(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusInternalServerError) + require.Equal(t, http.StatusInternalServerError, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Regexp(t, "^generic test error(?s:.)*$", string(data)) @@ -777,10 +803,10 @@ func TestRouteServiceStatusOK(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusOK) + require.Equal(t, http.StatusOK, resp.StatusCode) data := make(map[string]interface{}) dec := json.NewDecoder(resp.Body) err = dec.Decode(&data) @@ -817,10 +843,10 @@ func TestRouteServiceStatusNoDeployment(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusNotFound) + require.Equal(t, http.StatusNotFound, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Regexp(t, "^kube: no deployment(?s:.)*$", string(data)) @@ -868,10 +894,10 @@ func TestRouteServiceStatusKubernetesNotFound(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusNotFound) + require.Equal(t, http.StatusNotFound, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Regexp(t, "^fake error(?s:.)*$", string(data)) @@ -907,10 +933,10 @@ func TestRouteServiceStatusError(t *testing.T) { req.Header.Set("Content-Type", contentTypeJSON) - resp, err := test.gclient.hclient.Do(req) + resp, err := test.gwclient.hclient.Do(req) require.NoError(t, err) - require.Equal(t, resp.StatusCode, http.StatusInternalServerError) + require.Equal(t, http.StatusInternalServerError, resp.StatusCode) data, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Regexp(t, "^generic test error(?s:.)*$", string(data)) diff --git a/go.mod b/go.mod index de59164ad..5a3f688a8 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/akash-network/provider go 1.18 require ( - github.com/akash-network/akash-api v0.0.15 - github.com/akash-network/node v0.23.0-rc9 + github.com/akash-network/akash-api v0.0.19 + github.com/akash-network/node v0.23.0-rc20 github.com/avast/retry-go v3.0.0+incompatible github.com/boz/go-lifecycle v0.1.1-0.20190620234137-5139c86739b8 - github.com/cosmos/cosmos-sdk v0.45.15 + github.com/cosmos/cosmos-sdk v0.45.16 github.com/cskr/pubsub v1.0.2 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f github.com/go-kit/kit v0.12.0 @@ -97,7 +97,7 @@ require ( github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.5 // indirect - github.com/cosmos/ibc-go/v3 v3.4.0 // indirect + github.com/cosmos/ibc-go/v4 v4.4.1 // indirect github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect @@ -177,6 +177,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect github.com/minio/highwayhash v1.0.2 // indirect @@ -203,6 +204,7 @@ require ( github.com/rakyll/statik v0.1.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/cors v1.8.2 // indirect github.com/rs/zerolog v1.27.0 // indirect @@ -218,6 +220,7 @@ require ( github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tendermint/tm-db v0.6.7 // indirect + github.com/theckman/yacspin v0.13.12 // indirect github.com/tidwall/btree v1.5.0 // indirect github.com/zondax/hid v0.9.1 // indirect github.com/zondax/ledger-go v0.14.1 // indirect @@ -225,14 +228,14 @@ require ( go.step.sm/crypto v0.26.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.6.0 // indirect + golang.org/x/crypto v0.7.0 // indirect golang.org/x/exp v0.0.0-20221019170559-20944726eadf // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/term v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 32bae55f9..371c75839 100644 --- a/go.sum +++ b/go.sum @@ -181,16 +181,16 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/akash-network/akash-api v0.0.15 h1:J5d+8b5mEwd2EOrAevvEVP8mMTVihm8hZx4k0+EaIEs= -github.com/akash-network/akash-api v0.0.15/go.mod h1:e1QqkOFwxHKf88I3U5bPOmREdfHHHX2bY27ZZOFnTX4= +github.com/akash-network/akash-api v0.0.19 h1:D99DqD5ocBkepA5CmXJzdB1Jy0zA00pefdRDYdIQyHM= +github.com/akash-network/akash-api v0.0.19/go.mod h1:9/uYusyBcZecBQCgZWUbXRu0i1tyxj4/ze45XB2oLIU= github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0= github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo= github.com/akash-network/ledger-go v0.14.3/go.mod h1:NfsjfFvno9Kaq6mfpsKz4sqjnAVVEsVsnBJfKB4ueAs= github.com/akash-network/ledger-go/cosmos v0.14.3 h1:bEI9jLHM+Lm55idi4RfJlDez4/rVJs7E1MT0U2whYqI= github.com/akash-network/ledger-go/cosmos v0.14.3/go.mod h1:SjAfheQTE4rWk0ir+wjbOWxwj8nc8E4AZ08NdsvYG24= -github.com/akash-network/node v0.23.0-rc9 h1:CI7ippxpmKtzD/wRKu/4is0SIlY/tiaWfifjjUSHLTI= -github.com/akash-network/node v0.23.0-rc9/go.mod h1:IY5MshnhqlzzPWWigMHvhafF4tDqzkWMOCUtVOSygTo= +github.com/akash-network/node v0.23.0-rc20 h1:LLNzbDe149NhTCsphs5Om9WEr2aXVGWIJDHJ4SUh5HI= +github.com/akash-network/node v0.23.0-rc20/go.mod h1:wFBk1IXN/BZETOT1zXSl/J5Mo584kWS1nR9JrwtdPEM= github.com/alecthomas/participle/v2 v2.0.0-alpha7 h1:cK4vjj0VSgb3lN1nuKA5F7dw+1s1pWBe5bx7nNCnN+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -397,8 +397,8 @@ github.com/cosmos/cosmos-db v0.0.0-20221226095112-f3c38ecb5e32 h1:zlCp9n3uwQieEL github.com/cosmos/cosmos-db v0.0.0-20221226095112-f3c38ecb5e32/go.mod h1:kwMlEC4wWvB48zAShGKVqboJL6w4zCLesaNQ3YLU2BQ= github.com/cosmos/cosmos-proto v1.0.0-beta.1 h1:iDL5qh++NoXxG8hSy93FdYJut4XfgbShIocllGaXx/0= github.com/cosmos/cosmos-proto v1.0.0-beta.1/go.mod h1:8k2GNZghi5sDRFw/scPL8gMSowT1vDA+5ouxL8GjaUE= -github.com/cosmos/cosmos-sdk v0.45.15 h1:yyLZ9PylnR1DADf9FYGxnn8t3+Y5K2mMUXNUN38MI5A= -github.com/cosmos/cosmos-sdk v0.45.15/go.mod h1:bScuNwWAP0TZJpUf+SHXRU3xGoUPp+X9nAzfeIXts40= +github.com/cosmos/cosmos-sdk v0.45.16 h1:5ba/Bh5/LE55IwHQuCU4fiG4eXeDKtSWzehXRpaKDcw= +github.com/cosmos/cosmos-sdk v0.45.16/go.mod h1:bScuNwWAP0TZJpUf+SHXRU3xGoUPp+X9nAzfeIXts40= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= @@ -406,8 +406,8 @@ github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4 github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.5 h1:rGA3hOrgNxgRM5wYcSCxgQBap7fW82WZgY78V9po/iY= github.com/cosmos/iavl v0.19.5/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= -github.com/cosmos/ibc-go/v3 v3.4.0 h1:ha3cqEG36pqMWqA1D+kxDWBTZXpeFMd/aZIQF7I0xro= -github.com/cosmos/ibc-go/v3 v3.4.0/go.mod h1:VwB/vWu4ysT5DN2aF78d17LYmx3omSAdq6gpKvM7XRA= +github.com/cosmos/ibc-go/v4 v4.4.1 h1:pHPLEpQStGuHZe5J17WvG7w0VGwTmfsoAHrs45+vPfw= +github.com/cosmos/ibc-go/v4 v4.4.1/go.mod h1:UkKEQAPWckLuomhqG8XzeE5nWQPdiEYF8EIDWXBKSXA= github.com/cosmos/keyring v1.2.0 h1:8C1lBP9xhImmIabyXW4c3vFjjLiBdGCmfLUfeZlV1Yo= github.com/cosmos/keyring v1.2.0/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -1296,6 +1296,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= @@ -1591,6 +1593,8 @@ github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5 github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1725,6 +1729,8 @@ github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2l github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= +github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= +github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1895,8 +1901,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2013,8 +2019,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2035,8 +2041,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2159,14 +2165,14 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2178,8 +2184,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/make/codegen.mk b/make/codegen.mk index 3c49f7de9..1f0121902 100644 --- a/make/codegen.mk +++ b/make/codegen.mk @@ -4,6 +4,7 @@ generate: modvendor $(MOCKERY) .PHONY: kubetypes kubetypes: $(K8S_GENERATE_GROUPS) + rm -rf pkg/client/* GOBIN=$(AP_DEVCACHE_BIN) $(K8S_GENERATE_GROUPS) all \ github.com/akash-network/provider/pkg/client github.com/akash-network/provider/pkg/apis \ "akash.network:v2beta1,v2beta2" \ diff --git a/make/init.mk b/make/init.mk index 120d55abd..943c4368f 100644 --- a/make/init.mk +++ b/make/init.mk @@ -54,7 +54,17 @@ endif BINS := $(PROVIDER_SERVICES) akash -export GO := GO111MODULE=$(GO111MODULE) go +GOWORK ?= on + +GO_MOD ?= vendor +ifeq ($(GOWORK), on) +GO_MOD := readonly +endif + +export GO := GO111MODULE=$(GO111MODULE) go +GO_BUILD := $(GO) build -mod=$(GO_MOD) +GO_TEST := $(GO) test -mod=$(GO_MOD) +GO_VET := $(GO) vet -mod=$(GO_MOD) GO_MOD_NAME := $(shell go list -m 2>/dev/null) REPLACED_MODULES := $(shell go list -mod=readonly -m -f '{{ .Replace }}' all 2>/dev/null | grep -v -x -F "" | grep "^/") diff --git a/make/releasing.mk b/make/releasing.mk index 1e7e10d19..4999bfc7e 100644 --- a/make/releasing.mk +++ b/make/releasing.mk @@ -37,10 +37,10 @@ bins: $(BINS) .PHONY: build build: - $(GO) build -a ./... + $(GO_BUILD) -a ./... $(PROVIDER_SERVICES): modvendor - $(GO) build -o $@ $(BUILD_FLAGS) ./cmd/provider-services + $(GO_BUILD) -o $@ $(BUILD_FLAGS) ./cmd/provider-services .PHONY: provider-services provider-services: $(PROVIDER_SERVICES) diff --git a/make/test-integration.mk b/make/test-integration.mk index 4622f3fa8..2137449bd 100644 --- a/make/test-integration.mk +++ b/make/test-integration.mk @@ -2,6 +2,7 @@ BUILD_TAGS_K8S_INTEGRATION := k8s_integration BUILD_TAGS_E2E := e2e integration BUILD_TAGS_ALL := "$(BUILD_TAGS_K8S_INTEGRATION) $(BUILD_TAGS_E2E)" +TEST_MODULES ?= $(shell $(GO) list ./... | grep -v '/mocks\|/kubernetes_mock\|/pkg/client') # This is statically specified in the vagrant configuration # todo @troian check it still necessary @@ -18,15 +19,19 @@ test-e2e-integration: # ``` # KUSTOMIZE_INSTALLS=akash-operator-inventory make kube-cluster-setup-e2e # ``` - $(KIND_VARS) $(INTEGRATION_VARS) go test -count=1 -mod=readonly -p 4 -tags "e2e" -v ./integration/... -run TestIntegrationTestSuite -timeout 1500s + $(KIND_VARS) $(INTEGRATION_VARS) $(GO_TEST) -count=1 -p 4 -tags "e2e" -v ./integration/... -run TestIntegrationTestSuite -timeout 1500s .PHONY: test-e2e-integration-k8s test-e2e-integration-k8s: - $(INTEGRATION_VARS) KUBE_NODE_IP="$(KUBE_NODE_IP)" KUBE_INGRESS_IP=127.0.0.1 KUBE_INGRESS_PORT=10080 go test -count=1 -mod=readonly -p 4 -tags "e2e $(BUILD_MAINNET)" -v ./integration/... -run TestIntegrationTestSuite + $(INTEGRATION_VARS) \ + KUBE_NODE_IP="$(KUBE_NODE_IP)" \ + KUBE_INGRESS_IP=127.0.0.1 \ + KUBE_INGRESS_PORT=10080 \ + $(GO_TEST) -count=1 -p 4 -tags "e2e $(BUILD_MAINNET)" -v ./integration/... -run TestIntegrationTestSuite .PHONY: test-query-app test-query-app: - $(INTEGRATION_VARS) $(KIND_VARS) go test -mod=readonly -p 4 -tags "$(BUILD_TAGS_E2E)" -v ./integration/... -run TestQueryApp + $(INTEGRATION_VARS) $(KIND_VARS) $(GO_TEST) -p 4 -tags "$(BUILD_TAGS_E2E)" -v ./integration/... -run TestQueryApp .PHONY: test-k8s-integration test-k8s-integration: @@ -34,8 +39,8 @@ test-k8s-integration: # ``` # KUSTOMIZE_INSTALLS=akash-operator-inventory make kube-cluster-setup-e2e # ``` - go test -count=1 -v -tags "$(BUILD_TAGS_K8S_INTEGRATION)" ./pkg/apis/akash.network/v2beta2 - go test -count=1 -v -tags "$(BUILD_TAGS_K8S_INTEGRATION)" ./cluster/kube + $(GO_TEST) -count=1 -v -tags "$(BUILD_TAGS_K8S_INTEGRATION)" ./pkg/apis/akash.network/v2beta2 + $(GO_TEST) -count=1 -v -tags "$(BUILD_TAGS_K8S_INTEGRATION)" ./cluster/kube ############################################################################### @@ -52,15 +57,15 @@ shellcheck: .PHONY: test test: - $(GO) test -tags=$(BUILD_MAINNET) -timeout 300s ./... + $(GO_TEST) -tags=$(BUILD_MAINNET) -timeout 300s $(TEST_MODULES) .PHONY: test-nocache test-nocache: - $(GO) test -tags=$(BUILD_MAINNET) -count=1 ./... + $(GO_TEST) -tags=$(BUILD_MAINNET) -count=1 $(TEST_MODULES) .PHONY: test-full test-full: - $(GO) test -tags=$(BUILD_TAGS) -race ./... + $(GO_TEST) -tags=$(BUILD_TAGS) -race $(TEST_MODULES) .PHONY: test-coverage test-coverage: $(AP_DEVCACHE) @@ -68,4 +73,4 @@ test-coverage: $(AP_DEVCACHE) .PHONY: test-vet test-vet: - $(GO) vet ./... + $(GO_VET) ./... diff --git a/manifest/manager.go b/manifest/manager.go index 45eabf5ac..b19db7285 100644 --- a/manifest/manager.go +++ b/manifest/manager.go @@ -12,7 +12,7 @@ import ( "github.com/tendermint/tendermint/libs/log" - maniv2beta1 "github.com/akash-network/akash-api/go/manifest/v2beta2" + maniv2beta2 "github.com/akash-network/akash-api/go/manifest/v2beta2" dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" "github.com/akash-network/node/pubsub" @@ -79,7 +79,7 @@ type manager struct { data dtypes.QueryDeploymentResponse requests []manifestRequest pendingRequests []manifestRequest - manifests []*maniv2beta1.Manifest + manifests []*maniv2beta2.Manifest versions [][]byte localLeases []event.LeaseWon @@ -375,7 +375,7 @@ func (m *manager) validateRequests() { return } - manifests := make([]*maniv2beta1.Manifest, 0) + manifests := make([]*maniv2beta2.Manifest, 0) for _, req := range m.requests { // If the request context is complete then skip processing it select { @@ -431,11 +431,11 @@ func (m *manager) validateRequest(req manifestRequest) error { return ErrManifestVersion } - if err = maniv2beta1.ValidateManifest(req.value.Manifest); err != nil { + if err = maniv2beta2.ValidateManifest(req.value.Manifest); err != nil { return err } - if err = maniv2beta1.ValidateManifestWithDeployment(&req.value.Manifest, m.data.Groups); err != nil { + if err = maniv2beta2.ValidateManifestWithDeployment(&req.value.Manifest, m.data.Groups); err != nil { return err } @@ -452,7 +452,7 @@ func (m *manager) validateRequest(req manifestRequest) error { return nil } -func (m *manager) checkHostnamesForManifest(requestManifest maniv2beta1.Manifest, groupNames []string) error { +func (m *manager) checkHostnamesForManifest(requestManifest maniv2beta2.Manifest, groupNames []string) error { // Check if the hostnames are available. Do not block forever ownerAddr, err := m.data.GetDeployment().DeploymentID.GetOwnerAddress() if err != nil { diff --git a/mocks/client.go b/mocks/client.go index f37c4fb67..6e2637538 100644 --- a/mocks/client.go +++ b/mocks/client.go @@ -15,6 +15,8 @@ import ( provider "github.com/akash-network/provider" + types "github.com/cosmos/cosmos-sdk/types" + v1beta3 "github.com/akash-network/provider/cluster/types/v1beta3" ) @@ -257,23 +259,23 @@ func (_c *Client_Status_Call) RunAndReturn(run func(context.Context) (*provider. return _c } -// Validate provides a mock function with given fields: _a0, _a1 -func (_m *Client) Validate(_a0 context.Context, _a1 deploymentv1beta3.GroupSpec) (provider.ValidateGroupSpecResult, error) { - ret := _m.Called(_a0, _a1) +// Validate provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Client) Validate(_a0 context.Context, _a1 types.Address, _a2 deploymentv1beta3.GroupSpec) (provider.ValidateGroupSpecResult, error) { + ret := _m.Called(_a0, _a1, _a2) var r0 provider.ValidateGroupSpecResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, deploymentv1beta3.GroupSpec) (provider.ValidateGroupSpecResult, error)); ok { - return rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, types.Address, deploymentv1beta3.GroupSpec) (provider.ValidateGroupSpecResult, error)); ok { + return rf(_a0, _a1, _a2) } - if rf, ok := ret.Get(0).(func(context.Context, deploymentv1beta3.GroupSpec) provider.ValidateGroupSpecResult); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(context.Context, types.Address, deploymentv1beta3.GroupSpec) provider.ValidateGroupSpecResult); ok { + r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Get(0).(provider.ValidateGroupSpecResult) } - if rf, ok := ret.Get(1).(func(context.Context, deploymentv1beta3.GroupSpec) error); ok { - r1 = rf(_a0, _a1) + if rf, ok := ret.Get(1).(func(context.Context, types.Address, deploymentv1beta3.GroupSpec) error); ok { + r1 = rf(_a0, _a1, _a2) } else { r1 = ret.Error(1) } @@ -288,14 +290,15 @@ type Client_Validate_Call struct { // Validate is a helper method to define mock.On call // - _a0 context.Context -// - _a1 deploymentv1beta3.GroupSpec -func (_e *Client_Expecter) Validate(_a0 interface{}, _a1 interface{}) *Client_Validate_Call { - return &Client_Validate_Call{Call: _e.mock.On("Validate", _a0, _a1)} +// - _a1 types.Address +// - _a2 deploymentv1beta3.GroupSpec +func (_e *Client_Expecter) Validate(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Client_Validate_Call { + return &Client_Validate_Call{Call: _e.mock.On("Validate", _a0, _a1, _a2)} } -func (_c *Client_Validate_Call) Run(run func(_a0 context.Context, _a1 deploymentv1beta3.GroupSpec)) *Client_Validate_Call { +func (_c *Client_Validate_Call) Run(run func(_a0 context.Context, _a1 types.Address, _a2 deploymentv1beta3.GroupSpec)) *Client_Validate_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(deploymentv1beta3.GroupSpec)) + run(args[0].(context.Context), args[1].(types.Address), args[2].(deploymentv1beta3.GroupSpec)) }) return _c } @@ -305,7 +308,7 @@ func (_c *Client_Validate_Call) Return(_a0 provider.ValidateGroupSpecResult, _a1 return _c } -func (_c *Client_Validate_Call) RunAndReturn(run func(context.Context, deploymentv1beta3.GroupSpec) (provider.ValidateGroupSpecResult, error)) *Client_Validate_Call { +func (_c *Client_Validate_Call) RunAndReturn(run func(context.Context, types.Address, deploymentv1beta3.GroupSpec) (provider.ValidateGroupSpecResult, error)) *Client_Validate_Call { _c.Call.Return(run) return _c } diff --git a/pkg/apis/akash.network/crd.yaml b/pkg/apis/akash.network/crd.yaml index d60c5586b..38693f3cf 100644 --- a/pkg/apis/akash.network/crd.yaml +++ b/pkg/apis/akash.network/crd.yaml @@ -148,12 +148,45 @@ spec: type: boolean mount: type: string - provider_params: + scheduler_params: type: object - nullable: false + nullable: true properties: runtime_class: type: string + resources: + type: object + nullable: true + properties: + gpu: + type: object + nullable: true + properties: + vendor: + type: string + model: + type: string +# affinity: +# nullable: true +# type: object +# properties: +# node: +# nullable: true +# type: object +# properties: +# required: +# type: array +# items: +# type: object +# properties: +# key: +# type: string +# operator: +# type: string +# values: +# type: array +# items: +# type: string - name: v2beta1 served: true storage: false @@ -287,69 +320,6 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition -metadata: - # name must match the spec fields below, and be in the form: . - name: leaseparamsservices.akash.network - # DO NOT REMOVE resource-policy annotation! - annotations: - "helm.sh/resource-policy": keep -spec: - # group name to use for REST API: /apis// - group: akash.network - scope: Namespaced - names: - # plural name to be used in the URL: /apis/// - plural: leaseparamsservices - # singular name to be used as an alias on the CLI and for display - singular: leaseparamsservice - # kind is normally the CamelCased singular type. Your resource manifests use this. - kind: LeaseParamsService - # shortNames allow shorter string to match your resource on the CLI - shortNames: - - pls - # list of versions supported by this CustomResourceDefinition - versions: - - name: v2beta2 - # Each version can be enabled/disabled by Served flag. - served: true - # One and only one version must be marked as the storage version. - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - lease_id: - type: object - properties: - owner: - type: string - dseq: - type: string - format: uint64 - gseq: - type: integer - oseq: - type: integer - provider: - type: string - services: - type: array - items: - type: object - properties: - name: - type: string - params: - type: object - properties: - runtime_class: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition metadata: # name must match the spec fields below, and be in the form: . name: providerhosts.akash.network diff --git a/pkg/apis/akash.network/v2beta2/k8s_integration_test.go b/pkg/apis/akash.network/v2beta2/k8s_integration_test.go index e655970c5..f4048f97f 100644 --- a/pkg/apis/akash.network/v2beta2/k8s_integration_test.go +++ b/pkg/apis/akash.network/v2beta2/k8s_integration_test.go @@ -39,7 +39,10 @@ func TestWriteRead(t *testing.T) { lid := atestutil.LeaseID(t) group := spec.Generator.Group(t) - kmani, err := crd.NewManifest(ns, lid, &group) + csettings := crd.ClusterSettings{ + SchedulerParams: make([]*crd.SchedulerParams, len(group.Services)), + } + kmani, err := crd.NewManifest(ns, lid, &group, csettings) require.NoError(t, err, spec.Name) @@ -55,7 +58,7 @@ func TestWriteRead(t *testing.T) { require.NoError(t, err, spec.Name) assert.Equal(t, lid, deployment.LeaseID(), spec.Name) - assert.Equal(t, group, deployment.ManifestGroup(), spec.Name) + assert.Equal(t, &group, deployment.ManifestGroup(), spec.Name) } }) } diff --git a/pkg/apis/akash.network/v2beta2/lease_params_service.go b/pkg/apis/akash.network/v2beta2/lease_params_service.go deleted file mode 100644 index 9e69cfce8..000000000 --- a/pkg/apis/akash.network/v2beta2/lease_params_service.go +++ /dev/null @@ -1,45 +0,0 @@ -package v2beta2 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// LeaseParamsService -// +genclient -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -type LeaseParamsService struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata"` - - Spec LeaseParamsServiceSpec `json:"spec,omitempty"` - Status LeaseParamsServiceStatus `json:"status,omitempty"` -} - -// LeaseParamsServiceList -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -type LeaseParamsServiceList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata"` - Items []LeaseParamsService `json:"items"` -} - -type Params struct { - RuntimeClass string `json:"runtime_class"` -} - -type ParamsService struct { - Name string `json:"name"` - Params Params `json:"params"` -} - -type ParamsServices []ParamsService - -type LeaseParamsServiceSpec struct { - LeaseID LeaseID `json:"lease_id"` - Services ParamsServices `json:"services"` -} - -type LeaseParamsServiceStatus struct { - State string `json:"state,omitempty"` - Message string `json:"message,omitempty"` -} diff --git a/pkg/apis/akash.network/v2beta2/manifest.go b/pkg/apis/akash.network/v2beta2/manifest.go index 9a38b4ce7..b72d549ce 100644 --- a/pkg/apis/akash.network/v2beta2/manifest.go +++ b/pkg/apis/akash.network/v2beta2/manifest.go @@ -1,6 +1,8 @@ package v2beta2 import ( + "fmt" + mani "github.com/akash-network/akash-api/go/manifest/v2beta2" mtypes "github.com/akash-network/akash-api/go/node/market/v1beta3" types "github.com/akash-network/akash-api/go/node/types/v1beta3" @@ -46,7 +48,8 @@ type ManifestService struct { // Overlay Network Links Expose []ManifestServiceExpose `json:"expose,omitempty"` // Miscellaneous service parameters - Params *ManifestServiceParams `json:"params,omitempty"` + Params *ManifestServiceParams `json:"params,omitempty"` + SchedulerParams *SchedulerParams `json:"scheduler_params,omitempty"` } // ManifestGroup stores metadata, name and list of SDL manifest services @@ -79,6 +82,33 @@ type ManifestServiceParams struct { Storage []ManifestStorageParams `json:"storage,omitempty"` } +// type NodeAffinity struct { +// Required []corev1.NodeSelectorRequirement `json:"required"` +// } +// +// type Affinity struct { +// Node *NodeAffinity `json:"node"` +// } + +type SchedulerResourceGPU struct { + Vendor string `json:"vendor"` + Model string `json:"model"` +} + +type SchedulerResources struct { + GPU *SchedulerResourceGPU `json:"gpu"` +} + +type SchedulerParams struct { + RuntimeClass string `json:"runtime_class"` + Resources *SchedulerResources `json:"resources,omitempty"` + // Affinity *Affinity `json:"affinity"` +} + +type ClusterSettings struct { + SchedulerParams []*SchedulerParams `json:"scheduler_params"` +} + // ManifestServiceExpose stores exposed ports and accepted hosts details type ManifestServiceExpose struct { Port uint16 `json:"port,omitempty"` @@ -102,8 +132,16 @@ type ManifestServiceExposeHTTPOptions struct { } // NewManifest creates new manifest with provided details. Returns error in case of failure. -func NewManifest(ns string, lid mtypes.LeaseID, mgroup *mani.Group) (*Manifest, error) { - group, err := manifestGroupToCRD(mgroup) +func NewManifest(ns string, lid mtypes.LeaseID, mgroup *mani.Group, settings ClusterSettings) (*Manifest, error) { + if len(mgroup.Services) != len(settings.SchedulerParams) { + return nil, fmt.Errorf("%w: group services don't not match scheduler services count (%d) != (%d)", + ErrInvalidArgs, + len(mgroup.Services), + len(settings.SchedulerParams), + ) + } + + group, err := manifestGroupToCRD(mgroup, settings) if err != nil { return nil, err } @@ -125,46 +163,55 @@ func NewManifest(ns string, lid mtypes.LeaseID, mgroup *mani.Group) (*Manifest, } // Deployment returns the cluster.Deployment that the saved manifest represents. -func (m *Manifest) Deployment() (ctypes.Deployment, error) { +func (m *Manifest) Deployment() (ctypes.IDeployment, error) { lid, err := m.Spec.LeaseID.FromCRD() if err != nil { return nil, err } - group, err := m.Spec.Group.fromCRD() + group, schedulerParams, err := m.Spec.Group.fromCRD() if err != nil { return nil, err } - return deployment{lid: lid, group: group}, nil + + return &deployment{ + lid: lid, + group: group, + cparams: schedulerParams, + }, nil } // toAkash returns akash group details formatted from manifest group -func (m *ManifestGroup) fromCRD() (mani.Group, error) { +func (m *ManifestGroup) fromCRD() (mani.Group, []*SchedulerParams, error) { am := mani.Group{ Name: m.Name, Services: make([]mani.Service, 0, len(m.Services)), } + scheduleParams := make([]*SchedulerParams, 0, len(m.Services)) + for _, svc := range m.Services { asvc, err := svc.fromCRD() if err != nil { - return am, err + return am, nil, err } am.Services = append(am.Services, asvc) + + scheduleParams = append(scheduleParams, svc.SchedulerParams) } - return am, nil + return am, scheduleParams, nil } // manifestGroupToCRD returns manifest group instance from akash group -func manifestGroupToCRD(m *mani.Group) (ManifestGroup, error) { +func manifestGroupToCRD(m *mani.Group, settings ClusterSettings) (ManifestGroup, error) { ma := ManifestGroup{ Name: m.Name, Services: make([]ManifestService, 0, len(m.Services)), } - for _, svc := range m.Services { - service, err := manifestServiceFromProvider(svc) + for i, svc := range m.Services { + service, err := manifestServiceFromProvider(svc, settings.SchedulerParams[i]) if err != nil { return ManifestGroup{}, err } @@ -224,21 +271,22 @@ func (ms *ManifestService) fromCRD() (mani.Service, error) { return *ams, nil } -func manifestServiceFromProvider(ams mani.Service) (ManifestService, error) { +func manifestServiceFromProvider(ams mani.Service, schedulerParams *SchedulerParams) (ManifestService, error) { resources, err := resourceUnitsFromAkash(ams.Resources) if err != nil { return ManifestService{}, err } ms := ManifestService{ - Name: ams.Name, - Image: ams.Image, - Command: ams.Command, - Args: ams.Args, - Env: ams.Env, - Resources: resources, - Count: ams.Count, - Expose: make([]ManifestServiceExpose, 0, len(ams.Expose)), + Name: ams.Name, + Image: ams.Image, + Command: ams.Command, + Args: ams.Args, + Env: ams.Env, + Resources: resources, + Count: ams.Count, + Expose: make([]ManifestServiceExpose, 0, len(ams.Expose)), + SchedulerParams: schedulerParams, } for _, expose := range ams.Expose { diff --git a/pkg/apis/akash.network/v2beta2/node_info.go b/pkg/apis/akash.network/v2beta2/node_info.go new file mode 100644 index 000000000..09b036d6e --- /dev/null +++ b/pkg/apis/akash.network/v2beta2/node_info.go @@ -0,0 +1,24 @@ +package v2beta2 + +type GPUCapabilities struct { + Vendor string `json:"vendor" capabilities:"vendor"` + Model string `json:"string" capabilities:"model"` +} + +type StorageCapabilities struct { + Classes []string `json:"classes"` +} + +type NodeInfoCapabilities struct { + GPU GPUCapabilities `json:"gpu" capabilities:"gpu"` + Storage StorageCapabilities `json:"storage" capabilities:"storage"` +} + +func (c *StorageCapabilities) HasClass(class string) bool { + for _, val := range c.Classes { + if val == class { + return true + } + } + return false +} diff --git a/pkg/apis/akash.network/v2beta2/register.go b/pkg/apis/akash.network/v2beta2/register.go index 43b069ca6..26cab2471 100644 --- a/pkg/apis/akash.network/v2beta2/register.go +++ b/pkg/apis/akash.network/v2beta2/register.go @@ -26,10 +26,6 @@ func addKnownTypes(scheme *runtime.Scheme) error { &Manifest{}, &ManifestList{}, ) - scheme.AddKnownTypes(SchemeGroupVersion, - &LeaseParamsService{}, - &LeaseParamsServiceList{}) - scheme.AddKnownTypes(SchemeGroupVersion, &InventoryRequest{}, &InventoryRequestList{}, diff --git a/pkg/apis/akash.network/v2beta2/types.go b/pkg/apis/akash.network/v2beta2/types.go index a12bf3981..1775dfbbf 100644 --- a/pkg/apis/akash.network/v2beta2/types.go +++ b/pkg/apis/akash.network/v2beta2/types.go @@ -1,6 +1,7 @@ package v2beta2 import ( + "fmt" "math" "strconv" @@ -12,17 +13,31 @@ import ( types "github.com/akash-network/akash-api/go/node/types/v1beta3" ) +var ( + ErrInvalidArgs = fmt.Errorf("crd/%s: invalid args", crdVersion) +) + +type Status struct { + State string `json:"state,omitempty"` + Message string `json:"message,omitempty"` +} + type deployment struct { - lid mtypes.LeaseID - group mani.Group + lid mtypes.LeaseID + group mani.Group + cparams interface{} } func (d deployment) LeaseID() mtypes.LeaseID { return d.lid } -func (d deployment) ManifestGroup() mani.Group { - return d.group +func (d deployment) ManifestGroup() *mani.Group { + return &d.group +} + +func (d deployment) ClusterParams() interface{} { + return d.cparams } // LeaseID stores deployment, group sequence, order, provider and metadata diff --git a/pkg/apis/akash.network/v2beta2/types_test.go b/pkg/apis/akash.network/v2beta2/types_test.go index 0bbc52cfd..733314ada 100644 --- a/pkg/apis/akash.network/v2beta2/types_test.go +++ b/pkg/apis/akash.network/v2beta2/types_test.go @@ -17,14 +17,15 @@ func Test_Manifest_encoding(t *testing.T) { lid := atestutil.LeaseID(t) mgroup := spec.Generator.Group(t) + sparams := make([]*SchedulerParams, len(mgroup.Services)) - kmani, err := NewManifest("foo", lid, &mgroup) + kmani, err := NewManifest("foo", lid, &mgroup, ClusterSettings{SchedulerParams: sparams}) require.NoError(t, err, spec.Name) deployment, err := kmani.Deployment() require.NoError(t, err, spec.Name) assert.Equal(t, lid, deployment.LeaseID(), spec.Name) - assert.Equal(t, mgroup, deployment.ManifestGroup(), spec.Name) + assert.Equal(t, &mgroup, deployment.ManifestGroup(), spec.Name) } } diff --git a/pkg/apis/akash.network/v2beta2/zz_generated.deepcopy.go b/pkg/apis/akash.network/v2beta2/zz_generated.deepcopy.go index 28982e2ac..f39f0aef1 100644 --- a/pkg/apis/akash.network/v2beta2/zz_generated.deepcopy.go +++ b/pkg/apis/akash.network/v2beta2/zz_generated.deepcopy.go @@ -25,6 +25,49 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSettings) DeepCopyInto(out *ClusterSettings) { + *out = *in + if in.SchedulerParams != nil { + in, out := &in.SchedulerParams, &out.SchedulerParams + *out = make([]*SchedulerParams, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(SchedulerParams) + (*in).DeepCopyInto(*out) + } + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSettings. +func (in *ClusterSettings) DeepCopy() *ClusterSettings { + if in == nil { + return nil + } + out := new(ClusterSettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUCapabilities) DeepCopyInto(out *GPUCapabilities) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUCapabilities. +func (in *GPUCapabilities) DeepCopy() *GPUCapabilities { + if in == nil { + return nil + } + out := new(GPUCapabilities) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Inventory) DeepCopyInto(out *Inventory) { *out = *in @@ -254,105 +297,6 @@ func (in *LeaseID) DeepCopy() *LeaseID { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LeaseParamsService) DeepCopyInto(out *LeaseParamsService) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LeaseParamsService. -func (in *LeaseParamsService) DeepCopy() *LeaseParamsService { - if in == nil { - return nil - } - out := new(LeaseParamsService) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *LeaseParamsService) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LeaseParamsServiceList) DeepCopyInto(out *LeaseParamsServiceList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]LeaseParamsService, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LeaseParamsServiceList. -func (in *LeaseParamsServiceList) DeepCopy() *LeaseParamsServiceList { - if in == nil { - return nil - } - out := new(LeaseParamsServiceList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *LeaseParamsServiceList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LeaseParamsServiceSpec) DeepCopyInto(out *LeaseParamsServiceSpec) { - *out = *in - out.LeaseID = in.LeaseID - if in.Services != nil { - in, out := &in.Services, &out.Services - *out = make(ParamsServices, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LeaseParamsServiceSpec. -func (in *LeaseParamsServiceSpec) DeepCopy() *LeaseParamsServiceSpec { - if in == nil { - return nil - } - out := new(LeaseParamsServiceSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LeaseParamsServiceStatus) DeepCopyInto(out *LeaseParamsServiceStatus) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LeaseParamsServiceStatus. -func (in *LeaseParamsServiceStatus) DeepCopy() *LeaseParamsServiceStatus { - if in == nil { - return nil - } - out := new(LeaseParamsServiceStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Manifest) DeepCopyInto(out *Manifest) { *out = *in @@ -468,6 +412,11 @@ func (in *ManifestService) DeepCopyInto(out *ManifestService) { *out = new(ManifestServiceParams) (*in).DeepCopyInto(*out) } + if in.SchedulerParams != nil { + in, out := &in.SchedulerParams, &out.SchedulerParams + *out = new(SchedulerParams) + (*in).DeepCopyInto(*out) + } return } @@ -596,58 +545,23 @@ func (in *ManifestStorageParams) DeepCopy() *ManifestStorageParams { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Params) DeepCopyInto(out *Params) { +func (in *NodeInfoCapabilities) DeepCopyInto(out *NodeInfoCapabilities) { *out = *in + out.GPU = in.GPU + in.Storage.DeepCopyInto(&out.Storage) return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Params. -func (in *Params) DeepCopy() *Params { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeInfoCapabilities. +func (in *NodeInfoCapabilities) DeepCopy() *NodeInfoCapabilities { if in == nil { return nil } - out := new(Params) + out := new(NodeInfoCapabilities) in.DeepCopyInto(out) return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ParamsService) DeepCopyInto(out *ParamsService) { - *out = *in - out.Params = in.Params - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParamsService. -func (in *ParamsService) DeepCopy() *ParamsService { - if in == nil { - return nil - } - out := new(ParamsService) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ParamsServices) DeepCopyInto(out *ParamsServices) { - { - in := &in - *out = make(ParamsServices, len(*in)) - copy(*out, *in) - return - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ParamsServices. -func (in ParamsServices) DeepCopy() ParamsServices { - if in == nil { - return nil - } - out := new(ParamsServices) - in.DeepCopyInto(out) - return *out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProviderHost) DeepCopyInto(out *ProviderHost) { *out = *in @@ -887,3 +801,98 @@ func (in *ResourceUnitsStorage) DeepCopy() *ResourceUnitsStorage { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerParams) DeepCopyInto(out *SchedulerParams) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(SchedulerResources) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerParams. +func (in *SchedulerParams) DeepCopy() *SchedulerParams { + if in == nil { + return nil + } + out := new(SchedulerParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerResourceGPU) DeepCopyInto(out *SchedulerResourceGPU) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerResourceGPU. +func (in *SchedulerResourceGPU) DeepCopy() *SchedulerResourceGPU { + if in == nil { + return nil + } + out := new(SchedulerResourceGPU) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerResources) DeepCopyInto(out *SchedulerResources) { + *out = *in + if in.GPU != nil { + in, out := &in.GPU, &out.GPU + *out = new(SchedulerResourceGPU) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerResources. +func (in *SchedulerResources) DeepCopy() *SchedulerResources { + if in == nil { + return nil + } + out := new(SchedulerResources) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Status) DeepCopyInto(out *Status) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Status. +func (in *Status) DeepCopy() *Status { + if in == nil { + return nil + } + out := new(Status) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageCapabilities) DeepCopyInto(out *StorageCapabilities) { + *out = *in + if in.Classes != nil { + in, out := &in.Classes, &out.Classes + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageCapabilities. +func (in *StorageCapabilities) DeepCopy() *StorageCapabilities { + if in == nil { + return nil + } + out := new(StorageCapabilities) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/akash.network_client.go b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/akash.network_client.go index e7be2676b..e7470cd56 100644 --- a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/akash.network_client.go +++ b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/akash.network_client.go @@ -30,7 +30,6 @@ type AkashV2beta2Interface interface { RESTClient() rest.Interface InventoriesGetter InventoryRequestsGetter - LeaseParamsServicesGetter ManifestsGetter ProviderHostsGetter ProviderLeasedIPsGetter @@ -49,10 +48,6 @@ func (c *AkashV2beta2Client) InventoryRequests() InventoryRequestInterface { return newInventoryRequests(c) } -func (c *AkashV2beta2Client) LeaseParamsServices(namespace string) LeaseParamsServiceInterface { - return newLeaseParamsServices(c, namespace) -} - func (c *AkashV2beta2Client) Manifests(namespace string) ManifestInterface { return newManifests(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_akash.network_client.go b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_akash.network_client.go index 867b80987..34a8a12ce 100644 --- a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_akash.network_client.go +++ b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_akash.network_client.go @@ -36,10 +36,6 @@ func (c *FakeAkashV2beta2) InventoryRequests() v2beta2.InventoryRequestInterface return &FakeInventoryRequests{c} } -func (c *FakeAkashV2beta2) LeaseParamsServices(namespace string) v2beta2.LeaseParamsServiceInterface { - return &FakeLeaseParamsServices{c, namespace} -} - func (c *FakeAkashV2beta2) Manifests(namespace string) v2beta2.ManifestInterface { return &FakeManifests{c, namespace} } diff --git a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_leaseparamsservice.go b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_leaseparamsservice.go deleted file mode 100644 index a04234b24..000000000 --- a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/fake/fake_leaseparamsservice.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright The Akash Network Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - v2beta2 "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" -) - -// FakeLeaseParamsServices implements LeaseParamsServiceInterface -type FakeLeaseParamsServices struct { - Fake *FakeAkashV2beta2 - ns string -} - -var leaseparamsservicesResource = schema.GroupVersionResource{Group: "akash.network", Version: "v2beta2", Resource: "leaseparamsservices"} - -var leaseparamsservicesKind = schema.GroupVersionKind{Group: "akash.network", Version: "v2beta2", Kind: "LeaseParamsService"} - -// Get takes name of the leaseParamsService, and returns the corresponding leaseParamsService object, and an error if there is any. -func (c *FakeLeaseParamsServices) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2beta2.LeaseParamsService, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(leaseparamsservicesResource, c.ns, name), &v2beta2.LeaseParamsService{}) - - if obj == nil { - return nil, err - } - return obj.(*v2beta2.LeaseParamsService), err -} - -// List takes label and field selectors, and returns the list of LeaseParamsServices that match those selectors. -func (c *FakeLeaseParamsServices) List(ctx context.Context, opts v1.ListOptions) (result *v2beta2.LeaseParamsServiceList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(leaseparamsservicesResource, leaseparamsservicesKind, c.ns, opts), &v2beta2.LeaseParamsServiceList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v2beta2.LeaseParamsServiceList{ListMeta: obj.(*v2beta2.LeaseParamsServiceList).ListMeta} - for _, item := range obj.(*v2beta2.LeaseParamsServiceList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested leaseParamsServices. -func (c *FakeLeaseParamsServices) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(leaseparamsservicesResource, c.ns, opts)) - -} - -// Create takes the representation of a leaseParamsService and creates it. Returns the server's representation of the leaseParamsService, and an error, if there is any. -func (c *FakeLeaseParamsServices) Create(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.CreateOptions) (result *v2beta2.LeaseParamsService, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(leaseparamsservicesResource, c.ns, leaseParamsService), &v2beta2.LeaseParamsService{}) - - if obj == nil { - return nil, err - } - return obj.(*v2beta2.LeaseParamsService), err -} - -// Update takes the representation of a leaseParamsService and updates it. Returns the server's representation of the leaseParamsService, and an error, if there is any. -func (c *FakeLeaseParamsServices) Update(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.UpdateOptions) (result *v2beta2.LeaseParamsService, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(leaseparamsservicesResource, c.ns, leaseParamsService), &v2beta2.LeaseParamsService{}) - - if obj == nil { - return nil, err - } - return obj.(*v2beta2.LeaseParamsService), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeLeaseParamsServices) UpdateStatus(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.UpdateOptions) (*v2beta2.LeaseParamsService, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(leaseparamsservicesResource, "status", c.ns, leaseParamsService), &v2beta2.LeaseParamsService{}) - - if obj == nil { - return nil, err - } - return obj.(*v2beta2.LeaseParamsService), err -} - -// Delete takes name of the leaseParamsService and deletes it. Returns an error if one occurs. -func (c *FakeLeaseParamsServices) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(leaseparamsservicesResource, c.ns, name, opts), &v2beta2.LeaseParamsService{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeLeaseParamsServices) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(leaseparamsservicesResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &v2beta2.LeaseParamsServiceList{}) - return err -} - -// Patch applies the patch and returns the patched leaseParamsService. -func (c *FakeLeaseParamsServices) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2beta2.LeaseParamsService, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(leaseparamsservicesResource, c.ns, name, pt, data, subresources...), &v2beta2.LeaseParamsService{}) - - if obj == nil { - return nil, err - } - return obj.(*v2beta2.LeaseParamsService), err -} diff --git a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/generated_expansion.go b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/generated_expansion.go index 09ef02088..75cfc83c1 100644 --- a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/generated_expansion.go @@ -22,8 +22,6 @@ type InventoryExpansion interface{} type InventoryRequestExpansion interface{} -type LeaseParamsServiceExpansion interface{} - type ManifestExpansion interface{} type ProviderHostExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/leaseparamsservice.go b/pkg/client/clientset/versioned/typed/akash.network/v2beta2/leaseparamsservice.go deleted file mode 100644 index 64be937e1..000000000 --- a/pkg/client/clientset/versioned/typed/akash.network/v2beta2/leaseparamsservice.go +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright The Akash Network Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v2beta2 - -import ( - "context" - "time" - - v2beta2 "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" - scheme "github.com/akash-network/provider/pkg/client/clientset/versioned/scheme" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" -) - -// LeaseParamsServicesGetter has a method to return a LeaseParamsServiceInterface. -// A group's client should implement this interface. -type LeaseParamsServicesGetter interface { - LeaseParamsServices(namespace string) LeaseParamsServiceInterface -} - -// LeaseParamsServiceInterface has methods to work with LeaseParamsService resources. -type LeaseParamsServiceInterface interface { - Create(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.CreateOptions) (*v2beta2.LeaseParamsService, error) - Update(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.UpdateOptions) (*v2beta2.LeaseParamsService, error) - UpdateStatus(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.UpdateOptions) (*v2beta2.LeaseParamsService, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v2beta2.LeaseParamsService, error) - List(ctx context.Context, opts v1.ListOptions) (*v2beta2.LeaseParamsServiceList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2beta2.LeaseParamsService, err error) - LeaseParamsServiceExpansion -} - -// leaseParamsServices implements LeaseParamsServiceInterface -type leaseParamsServices struct { - client rest.Interface - ns string -} - -// newLeaseParamsServices returns a LeaseParamsServices -func newLeaseParamsServices(c *AkashV2beta2Client, namespace string) *leaseParamsServices { - return &leaseParamsServices{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the leaseParamsService, and returns the corresponding leaseParamsService object, and an error if there is any. -func (c *leaseParamsServices) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2beta2.LeaseParamsService, err error) { - result = &v2beta2.LeaseParamsService{} - err = c.client.Get(). - Namespace(c.ns). - Resource("leaseparamsservices"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of LeaseParamsServices that match those selectors. -func (c *leaseParamsServices) List(ctx context.Context, opts v1.ListOptions) (result *v2beta2.LeaseParamsServiceList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v2beta2.LeaseParamsServiceList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("leaseparamsservices"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested leaseParamsServices. -func (c *leaseParamsServices) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("leaseparamsservices"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a leaseParamsService and creates it. Returns the server's representation of the leaseParamsService, and an error, if there is any. -func (c *leaseParamsServices) Create(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.CreateOptions) (result *v2beta2.LeaseParamsService, err error) { - result = &v2beta2.LeaseParamsService{} - err = c.client.Post(). - Namespace(c.ns). - Resource("leaseparamsservices"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(leaseParamsService). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a leaseParamsService and updates it. Returns the server's representation of the leaseParamsService, and an error, if there is any. -func (c *leaseParamsServices) Update(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.UpdateOptions) (result *v2beta2.LeaseParamsService, err error) { - result = &v2beta2.LeaseParamsService{} - err = c.client.Put(). - Namespace(c.ns). - Resource("leaseparamsservices"). - Name(leaseParamsService.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(leaseParamsService). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *leaseParamsServices) UpdateStatus(ctx context.Context, leaseParamsService *v2beta2.LeaseParamsService, opts v1.UpdateOptions) (result *v2beta2.LeaseParamsService, err error) { - result = &v2beta2.LeaseParamsService{} - err = c.client.Put(). - Namespace(c.ns). - Resource("leaseparamsservices"). - Name(leaseParamsService.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(leaseParamsService). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the leaseParamsService and deletes it. Returns an error if one occurs. -func (c *leaseParamsServices) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("leaseparamsservices"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *leaseParamsServices) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("leaseparamsservices"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched leaseParamsService. -func (c *leaseParamsServices) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2beta2.LeaseParamsService, err error) { - result = &v2beta2.LeaseParamsService{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("leaseparamsservices"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/informers/externalversions/akash.network/v2beta2/interface.go b/pkg/client/informers/externalversions/akash.network/v2beta2/interface.go index ef083d672..98218b5fe 100644 --- a/pkg/client/informers/externalversions/akash.network/v2beta2/interface.go +++ b/pkg/client/informers/externalversions/akash.network/v2beta2/interface.go @@ -28,8 +28,6 @@ type Interface interface { Inventories() InventoryInformer // InventoryRequests returns a InventoryRequestInformer. InventoryRequests() InventoryRequestInformer - // LeaseParamsServices returns a LeaseParamsServiceInformer. - LeaseParamsServices() LeaseParamsServiceInformer // Manifests returns a ManifestInformer. Manifests() ManifestInformer // ProviderHosts returns a ProviderHostInformer. @@ -59,11 +57,6 @@ func (v *version) InventoryRequests() InventoryRequestInformer { return &inventoryRequestInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } -// LeaseParamsServices returns a LeaseParamsServiceInformer. -func (v *version) LeaseParamsServices() LeaseParamsServiceInformer { - return &leaseParamsServiceInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} - // Manifests returns a ManifestInformer. func (v *version) Manifests() ManifestInformer { return &manifestInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/akash.network/v2beta2/leaseparamsservice.go b/pkg/client/informers/externalversions/akash.network/v2beta2/leaseparamsservice.go deleted file mode 100644 index d34a6657f..000000000 --- a/pkg/client/informers/externalversions/akash.network/v2beta2/leaseparamsservice.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright The Akash Network Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package v2beta2 - -import ( - "context" - time "time" - - akashnetworkv2beta2 "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" - versioned "github.com/akash-network/provider/pkg/client/clientset/versioned" - internalinterfaces "github.com/akash-network/provider/pkg/client/informers/externalversions/internalinterfaces" - v2beta2 "github.com/akash-network/provider/pkg/client/listers/akash.network/v2beta2" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" -) - -// LeaseParamsServiceInformer provides access to a shared informer and lister for -// LeaseParamsServices. -type LeaseParamsServiceInformer interface { - Informer() cache.SharedIndexInformer - Lister() v2beta2.LeaseParamsServiceLister -} - -type leaseParamsServiceInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewLeaseParamsServiceInformer constructs a new informer for LeaseParamsService type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewLeaseParamsServiceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredLeaseParamsServiceInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredLeaseParamsServiceInformer constructs a new informer for LeaseParamsService type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredLeaseParamsServiceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options v1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.AkashV2beta2().LeaseParamsServices(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.AkashV2beta2().LeaseParamsServices(namespace).Watch(context.TODO(), options) - }, - }, - &akashnetworkv2beta2.LeaseParamsService{}, - resyncPeriod, - indexers, - ) -} - -func (f *leaseParamsServiceInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredLeaseParamsServiceInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *leaseParamsServiceInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&akashnetworkv2beta2.LeaseParamsService{}, f.defaultInformer) -} - -func (f *leaseParamsServiceInformer) Lister() v2beta2.LeaseParamsServiceLister { - return v2beta2.NewLeaseParamsServiceLister(f.Informer().GetIndexer()) -} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index c26decdf5..00ef08a71 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -70,8 +70,6 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Akash().V2beta2().Inventories().Informer()}, nil case v2beta2.SchemeGroupVersion.WithResource("inventoryrequests"): return &genericInformer{resource: resource.GroupResource(), informer: f.Akash().V2beta2().InventoryRequests().Informer()}, nil - case v2beta2.SchemeGroupVersion.WithResource("leaseparamsservices"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Akash().V2beta2().LeaseParamsServices().Informer()}, nil case v2beta2.SchemeGroupVersion.WithResource("manifests"): return &genericInformer{resource: resource.GroupResource(), informer: f.Akash().V2beta2().Manifests().Informer()}, nil case v2beta2.SchemeGroupVersion.WithResource("providerhosts"): diff --git a/pkg/client/listers/akash.network/v2beta2/expansion_generated.go b/pkg/client/listers/akash.network/v2beta2/expansion_generated.go index 47f329d09..54f1bd76a 100644 --- a/pkg/client/listers/akash.network/v2beta2/expansion_generated.go +++ b/pkg/client/listers/akash.network/v2beta2/expansion_generated.go @@ -26,14 +26,6 @@ type InventoryListerExpansion interface{} // InventoryRequestLister. type InventoryRequestListerExpansion interface{} -// LeaseParamsServiceListerExpansion allows custom methods to be added to -// LeaseParamsServiceLister. -type LeaseParamsServiceListerExpansion interface{} - -// LeaseParamsServiceNamespaceListerExpansion allows custom methods to be added to -// LeaseParamsServiceNamespaceLister. -type LeaseParamsServiceNamespaceListerExpansion interface{} - // ManifestListerExpansion allows custom methods to be added to // ManifestLister. type ManifestListerExpansion interface{} diff --git a/pkg/client/listers/akash.network/v2beta2/leaseparamsservice.go b/pkg/client/listers/akash.network/v2beta2/leaseparamsservice.go deleted file mode 100644 index 0ba750bf6..000000000 --- a/pkg/client/listers/akash.network/v2beta2/leaseparamsservice.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Copyright The Akash Network Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by lister-gen. DO NOT EDIT. - -package v2beta2 - -import ( - v2beta2 "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" -) - -// LeaseParamsServiceLister helps list LeaseParamsServices. -// All objects returned here must be treated as read-only. -type LeaseParamsServiceLister interface { - // List lists all LeaseParamsServices in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v2beta2.LeaseParamsService, err error) - // LeaseParamsServices returns an object that can list and get LeaseParamsServices. - LeaseParamsServices(namespace string) LeaseParamsServiceNamespaceLister - LeaseParamsServiceListerExpansion -} - -// leaseParamsServiceLister implements the LeaseParamsServiceLister interface. -type leaseParamsServiceLister struct { - indexer cache.Indexer -} - -// NewLeaseParamsServiceLister returns a new LeaseParamsServiceLister. -func NewLeaseParamsServiceLister(indexer cache.Indexer) LeaseParamsServiceLister { - return &leaseParamsServiceLister{indexer: indexer} -} - -// List lists all LeaseParamsServices in the indexer. -func (s *leaseParamsServiceLister) List(selector labels.Selector) (ret []*v2beta2.LeaseParamsService, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v2beta2.LeaseParamsService)) - }) - return ret, err -} - -// LeaseParamsServices returns an object that can list and get LeaseParamsServices. -func (s *leaseParamsServiceLister) LeaseParamsServices(namespace string) LeaseParamsServiceNamespaceLister { - return leaseParamsServiceNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// LeaseParamsServiceNamespaceLister helps list and get LeaseParamsServices. -// All objects returned here must be treated as read-only. -type LeaseParamsServiceNamespaceLister interface { - // List lists all LeaseParamsServices in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v2beta2.LeaseParamsService, err error) - // Get retrieves the LeaseParamsService from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v2beta2.LeaseParamsService, error) - LeaseParamsServiceNamespaceListerExpansion -} - -// leaseParamsServiceNamespaceLister implements the LeaseParamsServiceNamespaceLister -// interface. -type leaseParamsServiceNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all LeaseParamsServices in the indexer for a given namespace. -func (s leaseParamsServiceNamespaceLister) List(selector labels.Selector) (ret []*v2beta2.LeaseParamsService, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v2beta2.LeaseParamsService)) - }) - return ret, err -} - -// Get retrieves the LeaseParamsService from the indexer for a given namespace and name. -func (s leaseParamsServiceNamespaceLister) Get(name string) (*v2beta2.LeaseParamsService, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v2beta2.Resource("leaseparamsservice"), name) - } - return obj.(*v2beta2.LeaseParamsService), nil -} diff --git a/service.go b/service.go index 2725011b9..f2e624656 100644 --- a/service.go +++ b/service.go @@ -3,6 +3,7 @@ package provider import ( "context" + atypes "github.com/akash-network/akash-api/go/node/types/v1beta3" "github.com/boz/go-lifecycle" "github.com/pkg/errors" @@ -17,7 +18,7 @@ import ( "github.com/akash-network/provider/bidengine" "github.com/akash-network/provider/cluster" "github.com/akash-network/provider/cluster/operatorclients" - clustertypes "github.com/akash-network/provider/cluster/types/v1beta3" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" "github.com/akash-network/provider/manifest" "github.com/akash-network/provider/operator/waiter" "github.com/akash-network/provider/session" @@ -25,7 +26,7 @@ import ( // ValidateClient is the interface to check if provider will bid on given groupspec type ValidateClient interface { - Validate(context.Context, dtypes.GroupSpec) (ValidateGroupSpecResult, error) + Validate(context.Context, sdk.Address, dtypes.GroupSpec) (ValidateGroupSpecResult, error) } // StatusClient is the interface which includes status of service @@ -41,7 +42,7 @@ type Client interface { ValidateClient Manifest() manifest.Client Cluster() cluster.Client - Hostname() clustertypes.HostnameServiceClient + Hostname() ctypes.HostnameServiceClient ClusterService() cluster.Service } @@ -165,7 +166,7 @@ type service struct { lc lifecycle.Lifecycle } -func (s *service) Hostname() clustertypes.HostnameServiceClient { +func (s *service) Hostname() ctypes.HostnameServiceClient { return s.cluster.HostnameService() } @@ -211,9 +212,28 @@ func (s *service) Status(ctx context.Context) (*Status, error) { }, nil } -func (s *service) Validate(ctx context.Context, gspec dtypes.GroupSpec) (ValidateGroupSpecResult, error) { +func (s *service) Validate(ctx context.Context, owner sdk.Address, gspec dtypes.GroupSpec) (ValidateGroupSpecResult, error) { // FUTURE - pass owner here - price, err := s.config.BidPricingStrategy.CalculatePrice(ctx, "", &gspec) + req := bidengine.Request{ + Owner: owner.String(), + GSpec: &gspec, + } + + inv, err := s.cclient.Inventory(ctx) + if err != nil { + return ValidateGroupSpecResult{}, err + } + + res := &reservation{ + resources: nil, + clusterParams: nil, + } + + if err = inv.Adjust(res, ctypes.WithDryRun()); err != nil { + return ValidateGroupSpecResult{}, err + } + + price, err := s.config.BidPricingStrategy.CalculatePrice(ctx, req) if err != nil { return ValidateGroupSpecResult{}, err } @@ -246,3 +266,27 @@ func (s *service) run() { s.session.Log().Info("shutdown complete") } + +type reservation struct { + resources atypes.ResourceGroup + adjustedResources []atypes.Resources + clusterParams interface{} +} + +var _ ctypes.ReservationGroup = (*reservation)(nil) + +func (r *reservation) Resources() atypes.ResourceGroup { + return r.resources +} + +func (r *reservation) SetAllocatedResources(val []atypes.Resources) { + r.adjustedResources = val +} + +func (r *reservation) SetClusterParams(val interface{}) { + r.clusterParams = val +} + +func (r *reservation) ClusterParams() interface{} { + return r.clusterParams +}