From b2d7fb172313421375acdde597c48a70f5b5b804 Mon Sep 17 00:00:00 2001 From: grzywin Date: Tue, 28 Jan 2025 09:08:12 +0100 Subject: [PATCH] Make ScyllaDB and Scylla Manager flags configurable for e2e tests --- hack/.ci/lib/e2e.sh | 12 +++ hack/ci-deploy-release.sh | 45 +++++++++ hack/ci-deploy.sh | 13 +++ pkg/cmd/tests/options.go | 93 ++++++++++++++++--- test/e2e/framework/framework.go | 17 ++-- test/e2e/framework/testcontext.go | 23 +++-- .../scyllacluster/scyllacluster_upgrades.go | 21 ++--- 7 files changed, 182 insertions(+), 42 deletions(-) diff --git a/hack/.ci/lib/e2e.sh b/hack/.ci/lib/e2e.sh index 4fdc793db5c..a03c54e2ef0 100755 --- a/hack/.ci/lib/e2e.sh +++ b/hack/.ci/lib/e2e.sh @@ -207,6 +207,13 @@ function run-e2e { SO_BUCKET_NAME="${SO_BUCKET_NAME:-}" SO_E2E_PARALLELISM="${SO_E2E_PARALLELISM:-0}" + config_file="$(realpath "$(dirname "${BASH_SOURCE[0]}")/../../../assets/config/config.yaml")" + SCYLLADB_VERSION="${SCYLLADB_VERSION:-$(yq '.operator.scyllaDBVersion' "$config_file")}" + SCYLLADB_MANAGER_VERSION="${SCYLLADB_MANAGER_VERSION:-$(yq '.operator.scyllaDBManagerVersion' "$config_file")}" + SCYLLADB_MANAGER_AGENT_VERSION="${SCYLLADB_MANAGER_AGENT_VERSION:-$(yq '.operator.scyllaDBManagerAgentVersion' "$config_file")}" + SCYLLADB_UPDATE_FROM_VERSION="${SCYLLADB_UPDATE_FROM_VERSION:-$(yq '.operatorTests.scyllaDBVersions.updateFrom' "$config_file")}" + SCYLLADB_UPGRADE_FROM_VERSION="${SCYLLADB_UPGRADE_FROM_VERSION:-$(yq '.operatorTests.scyllaDBVersions.upgradeFrom' "$config_file")}" + kubectl create namespace e2e --dry-run=client -o=yaml | kubectl_create -f=- kubectl create clusterrolebinding e2e --clusterrole=cluster-admin --serviceaccount=e2e:default --dry-run=client -o=yaml | kubectl_create -f=- kubectl create -n=e2e pdb my-pdb --selector='app=e2e' --min-available=1 --dry-run=client -o=yaml | kubectl_create -f=- @@ -274,6 +281,11 @@ spec: - "--object-storage-bucket=${SO_BUCKET_NAME}" - "--gcs-service-account-key-path=${gcs_sa_in_container_path}" - "--s3-credentials-file-path=${s3_credentials_in_container_path}" + - "--scylladb-version=${SCYLLADB_VERSION}" + - "--scylladb-manager-version=${SCYLLADB_MANAGER_VERSION}" + - "--scylladb-manager-agent-version=${SCYLLADB_MANAGER_AGENT_VERSION}" + - "--scylladb-update-from-version=${SCYLLADB_UPDATE_FROM_VERSION}" + - "--scylladb-upgrade-from-version=${SCYLLADB_UPGRADE_FROM_VERSION}" image: "${SO_IMAGE}" imagePullPolicy: Always volumeMounts: diff --git a/hack/ci-deploy-release.sh b/hack/ci-deploy-release.sh index 4e7bcdbbc8f..0836c128bcf 100755 --- a/hack/ci-deploy-release.sh +++ b/hack/ci-deploy-release.sh @@ -275,6 +275,51 @@ target: EOF fi +if [[ -n "${SCYLLADB_VERSION:-}" ]]; then + cat << EOF | \ + yq eval-all --inplace 'select(fileIndex == 0) as $f | select(fileIndex == 1) as $p | with( $f.patches; . += $p | ... style="") | $f' "${ARTIFACTS_DEPLOY_DIR}/manager/kustomization.yaml" - +patch: |- + - op: replace + path: /spec/version + value: "${SCYLLADB_VERSION}" +target: + group: scylla.scylladb.com + version: v1 + kind: ScyllaCluster + name: scylla-manager-cluster +EOF +fi + +if [[ -n "${SCYLLA_MANAGER_VERSION:-}" ]]; then + cat << EOF | \ + yq eval-all --inplace 'select(fileIndex == 0) as $f | select(fileIndex == 1) as $p | with( $f.patches; . += $p | ... style="") | $f' "${ARTIFACTS_DEPLOY_DIR}/manager/kustomization.yaml" - +patch: |- + - op: replace + path: /spec/template/spec/containers/0/image + value: "docker.io/scylladb/scylla-manager:${SCYLLA_MANAGER_VERSION}" +target: + group: apps + version: v1 + kind: Deployment + name: scylla-manager +EOF +fi + +if [[ -n "${SCYLLA_MANAGER_AGENT_VERSION:-}" ]]; then + cat << EOF | \ + yq eval-all --inplace 'select(fileIndex == 0) as $f | select(fileIndex == 1) as $p | with( $f.patches; . += $p | ... style="") | $f' "${ARTIFACTS_DEPLOY_DIR}/manager/kustomization.yaml" - +patch: |- + - op: replace + path: /spec/agentVersion + value: "${SCYLLA_MANAGER_AGENT_VERSION}" +target: + group: scylla.scylladb.com + version: v1 + kind: ScyllaCluster + name: scylla-manager-cluster +EOF +fi + kubectl kustomize "${ARTIFACTS_DEPLOY_DIR}/manager" | kubectl_create -n=scylla-manager -f=- kubectl -n=scylla-manager wait --timeout=5m --for='condition=Progressing=False' scyllaclusters.scylla.scylladb.com/scylla-manager-cluster diff --git a/hack/ci-deploy.sh b/hack/ci-deploy.sh index ddcc90ffbb3..d55cccbfe04 100755 --- a/hack/ci-deploy.sh +++ b/hack/ci-deploy.sh @@ -113,6 +113,19 @@ if [[ -n "${SO_SCYLLACLUSTER_STORAGECLASS_NAME:-}" ]]; then elif [[ -n "${SO_SCYLLACLUSTER_STORAGECLASS_NAME+x}" ]]; then yq e --inplace 'del(.spec.datacenter.racks[0].storage.storageClassName)' "${DEPLOY_DIR}/manager/50_scyllacluster.yaml" fi + +if [[ -n "${SCYLLADB_VERSION:-}" ]]; then + yq e --inplace '.spec.version = env(SCYLLADB_VERSION)' "${DEPLOY_DIR}/manager/50_scyllacluster.yaml" +fi + +if [[ -n "${SCYLLA_MANAGER_VERSION:-}" ]]; then + yq e --inplace '.spec.template.spec.containers[0].image |= "docker.io/scylladb/scylla-manager:" + env(SCYLLA_MANAGER_VERSION)' "${DEPLOY_DIR}/manager/50_manager_deployment.yaml" +fi + +if [[ -n "${SCYLLA_MANAGER_AGENT_VERSION:-}" ]]; then + yq e --inplace '.spec.agentVersion = env(SCYLLA_MANAGER_AGENT_VERSION)' "${DEPLOY_DIR}/manager/50_scyllacluster.yaml" +fi + kubectl_create -f "${DEPLOY_DIR}"/manager kubectl -n=scylla-manager wait --timeout=10m --for='condition=Progressing=False' scyllaclusters.scylla.scylladb.com/scylla-manager-cluster diff --git a/pkg/cmd/tests/options.go b/pkg/cmd/tests/options.go index af673ddd666..5d733425e36 100644 --- a/pkg/cmd/tests/options.go +++ b/pkg/cmd/tests/options.go @@ -4,9 +4,11 @@ import ( "fmt" "os" "path" + "regexp" "strings" "github.com/onsi/ginkgo/v2" + configassets "github.com/scylladb/scylla-operator/assets/config" scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1" "github.com/scylladb/scylla-operator/pkg/genericclioptions" "github.com/scylladb/scylla-operator/pkg/helpers/slices" @@ -17,6 +19,16 @@ import ( "k8s.io/client-go/rest" ) +const ( + // https://github.com/distribution/reference/blob/8c942b0459dfdcc5b6685581dd0a5a470f615bff/regexp.go#L68 + referenceTagRegexp = `[\w][\w.-]{0,127}` + + // https://github.com/distribution/reference/blob/8c942b0459dfdcc5b6685581dd0a5a470f615bff/regexp.go#L81 + digestRegexp = `[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}` +) + +var tagWithOptionalDigestRegexp = regexp.MustCompile("^" + referenceTagRegexp + "(?:@" + digestRegexp + ")?$") + type IngressControllerOptions struct { Address string IngressClassName string @@ -55,6 +67,11 @@ type TestFrameworkOptions struct { objectStorageType framework.ObjectStorageType gcsServiceAccountKey []byte s3CredentialsFile []byte + ScyllaDBVersion string + ScyllaDBManagerVersion string + ScyllaDBManagerAgentVersion string + ScyllaDBUpdateFrom string + ScyllaDBUpgradeFrom string } func NewTestFrameworkOptions(streams genericclioptions.IOStreams, userAgent string) *TestFrameworkOptions { @@ -69,12 +86,17 @@ func NewTestFrameworkOptions(streams genericclioptions.IOStreams, userAgent stri ClientsBroadcastAddressType: string(scyllav1.BroadcastAddressTypePodIP), StorageClassName: "", }, - ObjectStorageBucket: "", - GCSServiceAccountKeyPath: "", - S3CredentialsFilePath: "", - objectStorageType: framework.ObjectStorageTypeNone, - gcsServiceAccountKey: []byte{}, - s3CredentialsFile: []byte{}, + ObjectStorageBucket: "", + GCSServiceAccountKeyPath: "", + S3CredentialsFilePath: "", + objectStorageType: framework.ObjectStorageTypeNone, + gcsServiceAccountKey: []byte{}, + s3CredentialsFile: []byte{}, + ScyllaDBVersion: configassets.Project.Operator.ScyllaDBVersion, + ScyllaDBManagerVersion: configassets.Project.Operator.ScyllaDBManagerVersion, + ScyllaDBManagerAgentVersion: configassets.Project.Operator.ScyllaDBManagerAgentVersion, + ScyllaDBUpdateFrom: configassets.Project.OperatorTests.ScyllaDBVersions.UpdateFrom, + ScyllaDBUpgradeFrom: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom, } } @@ -118,6 +140,11 @@ func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) { cmd.PersistentFlags().StringVarP(&o.ObjectStorageBucket, "object-storage-bucket", "", o.ObjectStorageBucket, "Name of the object storage bucket.") cmd.PersistentFlags().StringVarP(&o.GCSServiceAccountKeyPath, "gcs-service-account-key-path", "", o.GCSServiceAccountKeyPath, "Path to a file containing a GCS service account key.") cmd.PersistentFlags().StringVarP(&o.S3CredentialsFilePath, "s3-credentials-file-path", "", o.S3CredentialsFilePath, "Path to the AWS credentials file providing access to the S3 bucket.") + cmd.PersistentFlags().StringVarP(&o.ScyllaDBVersion, "scylladb-version", "", o.ScyllaDBVersion, "Version of ScyllaDB to use.") + cmd.PersistentFlags().StringVarP(&o.ScyllaDBManagerVersion, "scylladb-manager-version", "", o.ScyllaDBManagerVersion, "Version of Scylla Manager to use.") + cmd.PersistentFlags().StringVarP(&o.ScyllaDBManagerAgentVersion, "scylladb-manager-agent-version", "", o.ScyllaDBManagerAgentVersion, "Version of Scylla Manager Agent to use.") + cmd.PersistentFlags().StringVarP(&o.ScyllaDBUpdateFrom, "scylladb-update-from-version", "", o.ScyllaDBUpdateFrom, "Version of ScyllaDB to update from.") + cmd.PersistentFlags().StringVarP(&o.ScyllaDBUpgradeFrom, "scylladb-upgrade-from-version", "", o.ScyllaDBUpgradeFrom, "Version of ScyllaDB to upgrade from.") } func (o *TestFrameworkOptions) Validate(args []string) error { @@ -164,6 +191,41 @@ func (o *TestFrameworkOptions) Validate(args []string) error { errors = append(errors, fmt.Errorf("gcs-service-account-key-path and s3-credentials-file-path can't be set simultanously")) } + if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBVersion) { + errors = append(errors, fmt.Errorf( + "invalid scylladb-version format: %q. Expected format: [@]", + o.ScyllaDBVersion, + )) + } + + if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBUpdateFrom) { + errors = append(errors, fmt.Errorf( + "invalid scylladb-update-from-version format: %q. Expected format: [@]", + o.ScyllaDBUpdateFrom, + )) + } + + if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBUpgradeFrom) { + errors = append(errors, fmt.Errorf( + "invalid scylladb-upgrade-from-version format: %q. Expected format: [@]", + o.ScyllaDBUpgradeFrom, + )) + } + + if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBManagerVersion) { + errors = append(errors, fmt.Errorf( + "invalid scylladb-manager-version format: %q. Expected format: [@]", + o.ScyllaDBManagerVersion, + )) + } + + if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBManagerAgentVersion) { + errors = append(errors, fmt.Errorf( + "invalid scylladb-manager-agent-version format: %q. Expected format: [@]", + o.ScyllaDBManagerAgentVersion, + )) + } + if len(o.ArtifactsDir) > 0 { _, err = os.Stat(o.ArtifactsDir) if err != nil { @@ -226,13 +288,18 @@ func (o *TestFrameworkOptions) Complete(args []string) error { RestConfigs: slices.ConvertSlice(o.ClientConfigs, func(cc genericclioptions.ClientConfig) *rest.Config { return cc.RestConfig }), - ArtifactsDir: o.ArtifactsDir, - CleanupPolicy: o.CleanupPolicy, - ScyllaClusterOptions: o.scyllaClusterOptions, - ObjectStorageType: o.objectStorageType, - ObjectStorageBucket: o.ObjectStorageBucket, - GCSServiceAccountKey: o.gcsServiceAccountKey, - S3CredentialsFile: o.s3CredentialsFile, + ArtifactsDir: o.ArtifactsDir, + CleanupPolicy: o.CleanupPolicy, + ScyllaClusterOptions: o.scyllaClusterOptions, + ObjectStorageType: o.objectStorageType, + ObjectStorageBucket: o.ObjectStorageBucket, + GCSServiceAccountKey: o.gcsServiceAccountKey, + S3CredentialsFile: o.s3CredentialsFile, + ScyllaDBVersion: o.ScyllaDBVersion, + ScyllaDBManagerVersion: o.ScyllaDBManagerVersion, + ScyllaDBManagerAgentVersion: o.ScyllaDBManagerAgentVersion, + ScyllaDBUpdateFrom: o.ScyllaDBUpdateFrom, + ScyllaDBUpgradeFrom: o.ScyllaDBUpgradeFrom, } if o.IngressController != nil { diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index ca40f12f772..f78afe849af 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -12,7 +12,6 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" - configassets "github.com/scylladb/scylla-operator/assets/config" scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1" scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1" "github.com/scylladb/scylla-operator/pkg/controllerhelpers" @@ -135,8 +134,8 @@ func (f *Framework) CommonLabels() map[string]string { func (f *Framework) GetDefaultScyllaCluster() *scyllav1.ScyllaCluster { renderArgs := map[string]any{ - "scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion, - "scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerAgentVersion, + "scyllaDBVersion": TestContext.ScyllaDBVersion, + "scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion, "nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType, "nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType, "clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType, @@ -151,8 +150,8 @@ func (f *Framework) GetDefaultScyllaCluster() *scyllav1.ScyllaCluster { func (f *Framework) GetDefaultZonalScyllaClusterWithThreeRacks() *scyllav1.ScyllaCluster { renderArgs := map[string]any{ - "scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion, - "scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerAgentVersion, + "scyllaDBVersion": TestContext.ScyllaDBVersion, + "scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion, "nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType, "nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType, "clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType, @@ -168,8 +167,8 @@ func (f *Framework) GetDefaultZonalScyllaClusterWithThreeRacks() *scyllav1.Scyll func (f *Framework) GetDefaultScyllaDBDatacenter() *scyllav1alpha1.ScyllaDBDatacenter { renderArgs := map[string]any{ - "scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion, - "scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerVersion, + "scyllaDBVersion": TestContext.ScyllaDBVersion, + "scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion, "nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType, "nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType, "clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType, @@ -184,8 +183,8 @@ func (f *Framework) GetDefaultScyllaDBDatacenter() *scyllav1alpha1.ScyllaDBDatac func (f *Framework) GetDefaultScyllaDBCluster(rkcs []*scyllav1alpha1.RemoteKubernetesCluster) *scyllav1alpha1.ScyllaDBCluster { renderArgs := map[string]any{ - "scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion, - "scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerAgentVersion, + "scyllaDBVersion": TestContext.ScyllaDBVersion, + "scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion, "nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType, "nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType, "clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType, diff --git a/test/e2e/framework/testcontext.go b/test/e2e/framework/testcontext.go index effd0435de9..38ac88e2cdd 100644 --- a/test/e2e/framework/testcontext.go +++ b/test/e2e/framework/testcontext.go @@ -43,13 +43,18 @@ const ( ) type TestContextType struct { - RestConfigs []*restclient.Config - ArtifactsDir string - CleanupPolicy CleanupPolicyType - IngressController *IngressController - ScyllaClusterOptions *ScyllaClusterOptions - ObjectStorageType ObjectStorageType - ObjectStorageBucket string - GCSServiceAccountKey []byte - S3CredentialsFile []byte + RestConfigs []*restclient.Config + ArtifactsDir string + CleanupPolicy CleanupPolicyType + IngressController *IngressController + ScyllaClusterOptions *ScyllaClusterOptions + ObjectStorageType ObjectStorageType + ObjectStorageBucket string + GCSServiceAccountKey []byte + S3CredentialsFile []byte + ScyllaDBVersion string + ScyllaDBManagerVersion string + ScyllaDBManagerAgentVersion string + ScyllaDBUpdateFrom string + ScyllaDBUpgradeFrom string } diff --git a/test/e2e/set/scyllacluster/scyllacluster_upgrades.go b/test/e2e/set/scyllacluster/scyllacluster_upgrades.go index 0221d5b4e90..a62217b1241 100644 --- a/test/e2e/set/scyllacluster/scyllacluster_upgrades.go +++ b/test/e2e/set/scyllacluster/scyllacluster_upgrades.go @@ -8,7 +8,6 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" - configassets "github.com/scylladb/scylla-operator/assets/config" scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1" "github.com/scylladb/scylla-operator/pkg/controllerhelpers" "github.com/scylladb/scylla-operator/test/e2e/framework" @@ -115,32 +114,32 @@ var _ = g.Describe("ScyllaCluster upgrades", func() { g.Entry(describeEntry, &entry{ rackCount: 1, rackSize: 1, - initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpdateFrom, - updatedVersion: configassets.Project.Operator.ScyllaDBVersion, + initialVersion: framework.TestContext.ScyllaDBUpdateFrom, + updatedVersion: framework.TestContext.ScyllaDBVersion, }), g.Entry(describeEntry, &entry{ rackCount: 1, rackSize: 3, - initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpdateFrom, - updatedVersion: configassets.Project.Operator.ScyllaDBVersion, + initialVersion: framework.TestContext.ScyllaDBUpdateFrom, + updatedVersion: framework.TestContext.ScyllaDBVersion, }), g.Entry(describeEntry, &entry{ rackCount: 1, rackSize: 1, - initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom, - updatedVersion: configassets.Project.Operator.ScyllaDBVersion, + initialVersion: framework.TestContext.ScyllaDBUpgradeFrom, + updatedVersion: framework.TestContext.ScyllaDBVersion, }), g.Entry(describeEntry, &entry{ rackCount: 1, rackSize: 3, - initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom, - updatedVersion: configassets.Project.Operator.ScyllaDBVersion, + initialVersion: framework.TestContext.ScyllaDBUpgradeFrom, + updatedVersion: framework.TestContext.ScyllaDBVersion, }), g.Entry(describeEntry, &entry{ rackCount: 2, rackSize: 3, - initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom, - updatedVersion: configassets.Project.Operator.ScyllaDBVersion, + initialVersion: framework.TestContext.ScyllaDBUpgradeFrom, + updatedVersion: framework.TestContext.ScyllaDBVersion, }), ) })